diff options
Diffstat (limited to 'meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch')
-rw-r--r-- | meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch | 7171 |
1 files changed, 7171 insertions, 0 deletions
diff --git a/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch b/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch new file mode 100644 index 0000000000..3c7bfa160f --- /dev/null +++ b/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch | |||
@@ -0,0 +1,7171 @@ | |||
1 | From 27af8689a6ba8d182f3cbe6ba42cc654ceed0351 Mon Sep 17 00:00:00 2001 | ||
2 | From: Catalin Enache <catalin.enache@windriver.com> | ||
3 | Date: Fri, 30 Jun 2017 11:56:41 +0300 | ||
4 | Subject: [PATCH 4/6] New condvar implementation that provides stronger | ||
5 | ordering guarantees. | ||
6 | |||
7 | This is a new implementation for condition variables, required | ||
8 | after http://austingroupbugs.net/view.php?id=609 to fix bug 13165. In | ||
9 | essence, we need to be stricter in which waiters a signal or broadcast | ||
10 | is required to wake up; this couldn't be solved using the old algorithm. | ||
11 | ISO C++ made a similar clarification, so this also fixes a bug in | ||
12 | current libstdc++, for example. | ||
13 | |||
14 | We can't use the old algorithm anymore because futexes do not guarantee | ||
15 | to wake in FIFO order. Thus, when we wake, we can't simply let any | ||
16 | waiter grab a signal, but we need to ensure that one of the waiters | ||
17 | happening before the signal is woken up. This is something the previous | ||
18 | algorithm violated (see bug 13165). | ||
19 | |||
20 | There's another issue specific to condvars: ABA issues on the underlying | ||
21 | futexes. Unlike mutexes that have just three states, or semaphores that | ||
22 | have no tokens or a limited number of them, the state of a condvar is | ||
23 | the *order* of the waiters. A waiter on a semaphore can grab a token | ||
24 | whenever one is available; a condvar waiter must only consume a signal | ||
25 | if it is eligible to do so as determined by the relative order of the | ||
26 | waiter and the signal. | ||
27 | Therefore, this new algorithm maintains two groups of waiters: Those | ||
28 | eligible to consume signals (G1), and those that have to wait until | ||
29 | previous waiters have consumed signals (G2). Once G1 is empty, G2 | ||
30 | becomes the new G1. 64b counters are used to avoid ABA issues. | ||
31 | |||
32 | This condvar doesn't yet use a requeue optimization (ie, on a broadcast, | ||
33 | waking just one thread and requeueing all others on the futex of the | ||
34 | mutex supplied by the program). I don't think doing the requeue is | ||
35 | necessarily the right approach (but I haven't done real measurements | ||
36 | yet): | ||
37 | * If a program expects to wake many threads at the same time and make | ||
38 | that scalable, a condvar isn't great anyway because of how it requires | ||
39 | waiters to operate mutually exclusive (due to the mutex usage). Thus, a | ||
40 | thundering herd problem is a scalability problem with or without the | ||
41 | optimization. Using something like a semaphore might be more | ||
42 | appropriate in such a case. | ||
43 | * The scalability problem is actually at the mutex side; the condvar | ||
44 | could help (and it tries to with the requeue optimization), but it | ||
45 | should be the mutex who decides how that is done, and whether it is done | ||
46 | at all. | ||
47 | * Forcing all but one waiter into the kernel-side wait queue of the | ||
48 | mutex prevents/avoids the use of lock elision on the mutex. Thus, it | ||
49 | prevents the only cure against the underlying scalability problem | ||
50 | inherent to condvars. | ||
51 | * If condvars use short critical sections (ie, hold the mutex just to | ||
52 | check a binary flag or such), which they should do ideally, then forcing | ||
53 | all those waiter to proceed serially with kernel-based hand-off (ie, | ||
54 | futex ops in the mutex' contended state, via the futex wait queues) will | ||
55 | be less efficient than just letting a scalable mutex implementation take | ||
56 | care of it. Our current mutex impl doesn't employ spinning at all, but | ||
57 | if critical sections are short, spinning can be much better. | ||
58 | * Doing the requeue stuff requires all waiters to always drive the mutex | ||
59 | into the contended state. This leads to each waiter having to call | ||
60 | futex_wake after lock release, even if this wouldn't be necessary. | ||
61 | |||
62 | [BZ #13165] | ||
63 | * nptl/pthread_cond_broadcast.c (__pthread_cond_broadcast): Rewrite to | ||
64 | use new algorithm. | ||
65 | * nptl/pthread_cond_destroy.c (__pthread_cond_destroy): Likewise. | ||
66 | * nptl/pthread_cond_init.c (__pthread_cond_init): Likewise. | ||
67 | * nptl/pthread_cond_signal.c (__pthread_cond_signal): Likewise. | ||
68 | * nptl/pthread_cond_wait.c (__pthread_cond_wait): Likewise. | ||
69 | (__pthread_cond_timedwait): Move here from pthread_cond_timedwait.c. | ||
70 | (__condvar_confirm_wakeup, __condvar_cancel_waiting, | ||
71 | __condvar_cleanup_waiting, __condvar_dec_grefs, | ||
72 | __pthread_cond_wait_common): New. | ||
73 | (__condvar_cleanup): Remove. | ||
74 | * npt/pthread_condattr_getclock.c (pthread_condattr_getclock): Adapt. | ||
75 | * npt/pthread_condattr_setclock.c (pthread_condattr_setclock): | ||
76 | Likewise. | ||
77 | * npt/pthread_condattr_getpshared.c (pthread_condattr_getpshared): | ||
78 | Likewise. | ||
79 | * npt/pthread_condattr_init.c (pthread_condattr_init): Likewise. | ||
80 | * nptl/tst-cond1.c: Add comment. | ||
81 | * nptl/tst-cond20.c (do_test): Adapt. | ||
82 | * nptl/tst-cond22.c (do_test): Likewise. | ||
83 | * sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_cond_t): Adapt | ||
84 | structure. | ||
85 | * sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
86 | * sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
87 | * sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
88 | * sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_cond_t): | ||
89 | Likewise. | ||
90 | * sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
91 | * sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
92 | * sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
93 | * sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
94 | * sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
95 | * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_cond_t): | ||
96 | Likewise. | ||
97 | * sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h (pthread_cond_t): | ||
98 | Likewise. | ||
99 | * sysdeps/x86/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
100 | * sysdeps/nptl/internaltypes.h (COND_NWAITERS_SHIFT): Remove. | ||
101 | (COND_CLOCK_BITS): Adapt. | ||
102 | * sysdeps/nptl/pthread.h (PTHREAD_COND_INITIALIZER): Adapt. | ||
103 | * nptl/pthreadP.h (__PTHREAD_COND_CLOCK_MONOTONIC_MASK, | ||
104 | __PTHREAD_COND_SHARED_MASK): New. | ||
105 | * nptl/nptl-printers.py (CLOCK_IDS): Remove. | ||
106 | (ConditionVariablePrinter, ConditionVariableAttributesPrinter): Adapt. | ||
107 | * nptl/nptl_lock_constants.pysym: Adapt. | ||
108 | * nptl/test-cond-printers.py: Adapt. | ||
109 | * sysdeps/unix/sysv/linux/hppa/internaltypes.h (cond_compat_clear, | ||
110 | cond_compat_check_and_clear): Adapt. | ||
111 | * sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c: Remove file ... | ||
112 | * sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | ||
113 | (__pthread_cond_timedwait): ... and move here. | ||
114 | * nptl/DESIGN-condvar.txt: Remove file. | ||
115 | * nptl/lowlevelcond.sym: Likewise. | ||
116 | * nptl/pthread_cond_timedwait.c: Likewise. | ||
117 | * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise. | ||
118 | * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise. | ||
119 | * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise. | ||
120 | * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise. | ||
121 | * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_broadcast.S: Likewise. | ||
122 | * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_signal.S: Likewise. | ||
123 | * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_timedwait.S: Likewise. | ||
124 | * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_wait.S: Likewise. | ||
125 | * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_broadcast.S: Likewise. | ||
126 | * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_signal.S: Likewise. | ||
127 | * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S: Likewise. | ||
128 | * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_wait.S: Likewise. | ||
129 | * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise. | ||
130 | * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise. | ||
131 | * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. | ||
132 | * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise. | ||
133 | |||
134 | Upstream-Status: Backport | ||
135 | |||
136 | Author: Torvald Riegel <triegel@redhat.com> | ||
137 | Signed-off-by: Catalin Enache <catalin.enache@windriver.com> | ||
138 | --- | ||
139 | ChangeLog | 74 ++ | ||
140 | nptl/DESIGN-condvar.txt | 134 --- | ||
141 | nptl/Makefile | 6 +- | ||
142 | nptl/lowlevelcond.sym | 16 - | ||
143 | nptl/nptl-printers.py | 70 +- | ||
144 | nptl/nptl_lock_constants.pysym | 27 +- | ||
145 | nptl/pthreadP.h | 7 + | ||
146 | nptl/pthread_cond_broadcast.c | 99 ++- | ||
147 | nptl/pthread_cond_common.c | 466 ++++++++++ | ||
148 | nptl/pthread_cond_destroy.c | 82 +- | ||
149 | nptl/pthread_cond_init.c | 28 +- | ||
150 | nptl/pthread_cond_signal.c | 99 ++- | ||
151 | nptl/pthread_cond_timedwait.c | 268 ------ | ||
152 | nptl/pthread_cond_wait.c | 754 ++++++++++++---- | ||
153 | nptl/pthread_condattr_getclock.c | 2 +- | ||
154 | nptl/pthread_condattr_getpshared.c | 3 +- | ||
155 | nptl/pthread_condattr_init.c | 4 +- | ||
156 | nptl/pthread_condattr_setclock.c | 11 +- | ||
157 | nptl/test-cond-printers.py | 2 +- | ||
158 | nptl/tst-cond1.c | 3 + | ||
159 | nptl/tst-cond20.c | 5 +- | ||
160 | nptl/tst-cond22.c | 18 +- | ||
161 | sysdeps/aarch64/nptl/bits/pthreadtypes.h | 31 +- | ||
162 | sysdeps/arm/nptl/bits/pthreadtypes.h | 29 +- | ||
163 | sysdeps/ia64/nptl/bits/pthreadtypes.h | 31 +- | ||
164 | sysdeps/m68k/nptl/bits/pthreadtypes.h | 32 +- | ||
165 | sysdeps/microblaze/nptl/bits/pthreadtypes.h | 29 +- | ||
166 | sysdeps/mips/nptl/bits/pthreadtypes.h | 31 +- | ||
167 | sysdeps/nios2/nptl/bits/pthreadtypes.h | 31 +- | ||
168 | sysdeps/nptl/internaltypes.h | 17 +- | ||
169 | sysdeps/nptl/pthread.h | 2 +- | ||
170 | sysdeps/s390/nptl/bits/pthreadtypes.h | 29 +- | ||
171 | sysdeps/sh/nptl/bits/pthreadtypes.h | 29 +- | ||
172 | sysdeps/tile/nptl/bits/pthreadtypes.h | 29 +- | ||
173 | sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h | 31 +- | ||
174 | sysdeps/unix/sysv/linux/hppa/internaltypes.h | 40 +- | ||
175 | .../unix/sysv/linux/hppa/pthread_cond_timedwait.c | 41 - | ||
176 | sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | 13 + | ||
177 | .../sysv/linux/i386/i686/pthread_cond_timedwait.S | 20 - | ||
178 | .../unix/sysv/linux/i386/pthread_cond_broadcast.S | 241 ----- | ||
179 | sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S | 216 ----- | ||
180 | .../unix/sysv/linux/i386/pthread_cond_timedwait.S | 974 --------------------- | ||
181 | sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S | 642 -------------- | ||
182 | .../unix/sysv/linux/powerpc/bits/pthreadtypes.h | 31 +- | ||
183 | .../sysv/linux/x86_64/pthread_cond_broadcast.S | 177 ---- | ||
184 | .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 161 ---- | ||
185 | .../sysv/linux/x86_64/pthread_cond_timedwait.S | 623 ------------- | ||
186 | sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | 555 ------------ | ||
187 | sysdeps/x86/bits/pthreadtypes.h | 29 +- | ||
188 | 49 files changed, 1671 insertions(+), 4621 deletions(-) | ||
189 | delete mode 100644 nptl/DESIGN-condvar.txt | ||
190 | delete mode 100644 nptl/lowlevelcond.sym | ||
191 | create mode 100644 nptl/pthread_cond_common.c | ||
192 | delete mode 100644 nptl/pthread_cond_timedwait.c | ||
193 | delete mode 100644 sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c | ||
194 | delete mode 100644 sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S | ||
195 | delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S | ||
196 | delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S | ||
197 | delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S | ||
198 | delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S | ||
199 | delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S | ||
200 | delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S | ||
201 | delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S | ||
202 | delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | ||
203 | |||
204 | diff --git a/ChangeLog b/ChangeLog | ||
205 | index 8036c1e..c94db7b 100644 | ||
206 | --- a/ChangeLog | ||
207 | +++ b/ChangeLog | ||
208 | @@ -1,3 +1,77 @@ | ||
209 | +2016-12-31 Torvald Riegel <triegel@redhat.com> | ||
210 | + | ||
211 | + [BZ #13165] | ||
212 | + * nptl/pthread_cond_broadcast.c (__pthread_cond_broadcast): Rewrite to | ||
213 | + use new algorithm. | ||
214 | + * nptl/pthread_cond_destroy.c (__pthread_cond_destroy): Likewise. | ||
215 | + * nptl/pthread_cond_init.c (__pthread_cond_init): Likewise. | ||
216 | + * nptl/pthread_cond_signal.c (__pthread_cond_signal): Likewise. | ||
217 | + * nptl/pthread_cond_wait.c (__pthread_cond_wait): Likewise. | ||
218 | + (__pthread_cond_timedwait): Move here from pthread_cond_timedwait.c. | ||
219 | + (__condvar_confirm_wakeup, __condvar_cancel_waiting, | ||
220 | + __condvar_cleanup_waiting, __condvar_dec_grefs, | ||
221 | + __pthread_cond_wait_common): New. | ||
222 | + (__condvar_cleanup): Remove. | ||
223 | + * npt/pthread_condattr_getclock.c (pthread_condattr_getclock): Adapt. | ||
224 | + * npt/pthread_condattr_setclock.c (pthread_condattr_setclock): | ||
225 | + Likewise. | ||
226 | + * npt/pthread_condattr_getpshared.c (pthread_condattr_getpshared): | ||
227 | + Likewise. | ||
228 | + * npt/pthread_condattr_init.c (pthread_condattr_init): Likewise. | ||
229 | + * nptl/tst-cond1.c: Add comment. | ||
230 | + * nptl/tst-cond20.c (do_test): Adapt. | ||
231 | + * nptl/tst-cond22.c (do_test): Likewise. | ||
232 | + * sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_cond_t): Adapt | ||
233 | + structure. | ||
234 | + * sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
235 | + * sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
236 | + * sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
237 | + * sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_cond_t): | ||
238 | + Likewise. | ||
239 | + * sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
240 | + * sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
241 | + * sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
242 | + * sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
243 | + * sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
244 | + * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_cond_t): | ||
245 | + Likewise. | ||
246 | + * sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h (pthread_cond_t): | ||
247 | + Likewise. | ||
248 | + * sysdeps/x86/bits/pthreadtypes.h (pthread_cond_t): Likewise. | ||
249 | + * sysdeps/nptl/internaltypes.h (COND_NWAITERS_SHIFT): Remove. | ||
250 | + (COND_CLOCK_BITS): Adapt. | ||
251 | + * sysdeps/nptl/pthread.h (PTHREAD_COND_INITIALIZER): Adapt. | ||
252 | + * nptl/pthreadP.h (__PTHREAD_COND_CLOCK_MONOTONIC_MASK, | ||
253 | + __PTHREAD_COND_SHARED_MASK): New. | ||
254 | + * nptl/nptl-printers.py (CLOCK_IDS): Remove. | ||
255 | + (ConditionVariablePrinter, ConditionVariableAttributesPrinter): Adapt. | ||
256 | + * nptl/nptl_lock_constants.pysym: Adapt. | ||
257 | + * nptl/test-cond-printers.py: Adapt. | ||
258 | + * sysdeps/unix/sysv/linux/hppa/internaltypes.h (cond_compat_clear, | ||
259 | + cond_compat_check_and_clear): Adapt. | ||
260 | + * sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c: Remove file ... | ||
261 | + * sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | ||
262 | + (__pthread_cond_timedwait): ... and move here. | ||
263 | + * nptl/DESIGN-condvar.txt: Remove file. | ||
264 | + * nptl/lowlevelcond.sym: Likewise. | ||
265 | + * nptl/pthread_cond_timedwait.c: Likewise. | ||
266 | + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise. | ||
267 | + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise. | ||
268 | + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise. | ||
269 | + * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise. | ||
270 | + * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_broadcast.S: Likewise. | ||
271 | + * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_signal.S: Likewise. | ||
272 | + * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_timedwait.S: Likewise. | ||
273 | + * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_wait.S: Likewise. | ||
274 | + * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_broadcast.S: Likewise. | ||
275 | + * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_signal.S: Likewise. | ||
276 | + * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S: Likewise. | ||
277 | + * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_wait.S: Likewise. | ||
278 | + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise. | ||
279 | + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise. | ||
280 | + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise. | ||
281 | + * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise. | ||
282 | + | ||
283 | 2016-12-08 Martin Galvan <martin.galvan@tallertechnologies.com> | ||
284 | |||
285 | * INSTALL: Regenerated. | ||
286 | diff --git a/nptl/DESIGN-condvar.txt b/nptl/DESIGN-condvar.txt | ||
287 | deleted file mode 100644 | ||
288 | index 4845251..0000000 | ||
289 | --- a/nptl/DESIGN-condvar.txt | ||
290 | +++ /dev/null | ||
291 | @@ -1,134 +0,0 @@ | ||
292 | -Conditional Variable pseudocode. | ||
293 | -================================ | ||
294 | - | ||
295 | - int pthread_cond_timedwait (pthread_cond_t *cv, pthread_mutex_t *mutex); | ||
296 | - int pthread_cond_signal (pthread_cond_t *cv); | ||
297 | - int pthread_cond_broadcast (pthread_cond_t *cv); | ||
298 | - | ||
299 | -struct pthread_cond_t { | ||
300 | - | ||
301 | - unsigned int cond_lock; | ||
302 | - | ||
303 | - internal mutex | ||
304 | - | ||
305 | - uint64_t total_seq; | ||
306 | - | ||
307 | - Total number of threads using the conditional variable. | ||
308 | - | ||
309 | - uint64_t wakeup_seq; | ||
310 | - | ||
311 | - sequence number for next wakeup. | ||
312 | - | ||
313 | - uint64_t woken_seq; | ||
314 | - | ||
315 | - sequence number of last woken thread. | ||
316 | - | ||
317 | - uint32_t broadcast_seq; | ||
318 | - | ||
319 | -} | ||
320 | - | ||
321 | - | ||
322 | -struct cv_data { | ||
323 | - | ||
324 | - pthread_cond_t *cv; | ||
325 | - | ||
326 | - uint32_t bc_seq | ||
327 | - | ||
328 | -} | ||
329 | - | ||
330 | - | ||
331 | - | ||
332 | -cleanup_handler(cv_data) | ||
333 | -{ | ||
334 | - cv = cv_data->cv; | ||
335 | - lll_lock(cv->lock); | ||
336 | - | ||
337 | - if (cv_data->bc_seq == cv->broadcast_seq) { | ||
338 | - ++cv->wakeup_seq; | ||
339 | - ++cv->woken_seq; | ||
340 | - } | ||
341 | - | ||
342 | - /* make sure no signal gets lost. */ | ||
343 | - FUTEX_WAKE(cv->wakeup_seq, ALL); | ||
344 | - | ||
345 | - lll_unlock(cv->lock); | ||
346 | -} | ||
347 | - | ||
348 | - | ||
349 | -cond_timedwait(cv, mutex, timeout): | ||
350 | -{ | ||
351 | - lll_lock(cv->lock); | ||
352 | - mutex_unlock(mutex); | ||
353 | - | ||
354 | - cleanup_push | ||
355 | - | ||
356 | - ++cv->total_seq; | ||
357 | - val = seq = cv->wakeup_seq; | ||
358 | - cv_data.bc = cv->broadcast_seq; | ||
359 | - cv_data.cv = cv; | ||
360 | - | ||
361 | - while (1) { | ||
362 | - | ||
363 | - lll_unlock(cv->lock); | ||
364 | - | ||
365 | - enable_async(&cv_data); | ||
366 | - | ||
367 | - ret = FUTEX_WAIT(cv->wakeup_seq, val, timeout); | ||
368 | - | ||
369 | - restore_async | ||
370 | - | ||
371 | - lll_lock(cv->lock); | ||
372 | - | ||
373 | - if (bc != cv->broadcast_seq) | ||
374 | - goto bc_out; | ||
375 | - | ||
376 | - val = cv->wakeup_seq; | ||
377 | - | ||
378 | - if (val != seq && cv->woken_seq != val) { | ||
379 | - ret = 0; | ||
380 | - break; | ||
381 | - } | ||
382 | - | ||
383 | - if (ret == TIMEDOUT) { | ||
384 | - ++cv->wakeup_seq; | ||
385 | - break; | ||
386 | - } | ||
387 | - } | ||
388 | - | ||
389 | - ++cv->woken_seq; | ||
390 | - | ||
391 | - bc_out: | ||
392 | - lll_unlock(cv->lock); | ||
393 | - | ||
394 | - cleanup_pop | ||
395 | - | ||
396 | - mutex_lock(mutex); | ||
397 | - | ||
398 | - return ret; | ||
399 | -} | ||
400 | - | ||
401 | -cond_signal(cv) | ||
402 | -{ | ||
403 | - lll_lock(cv->lock); | ||
404 | - | ||
405 | - if (cv->total_seq > cv->wakeup_seq) { | ||
406 | - ++cv->wakeup_seq; | ||
407 | - FUTEX_WAKE(cv->wakeup_seq, 1); | ||
408 | - } | ||
409 | - | ||
410 | - lll_unlock(cv->lock); | ||
411 | -} | ||
412 | - | ||
413 | -cond_broadcast(cv) | ||
414 | -{ | ||
415 | - lll_lock(cv->lock); | ||
416 | - | ||
417 | - if (cv->total_seq > cv->wakeup_seq) { | ||
418 | - cv->wakeup_seq = cv->total_seq; | ||
419 | - cv->woken_seq = cv->total_seq; | ||
420 | - ++cv->broadcast_seq; | ||
421 | - FUTEX_WAKE(cv->wakeup_seq, ALL); | ||
422 | - } | ||
423 | - | ||
424 | - lll_unlock(cv->lock); | ||
425 | -} | ||
426 | diff --git a/nptl/Makefile b/nptl/Makefile | ||
427 | index 49f6ba6..1f0674c 100644 | ||
428 | --- a/nptl/Makefile | ||
429 | +++ b/nptl/Makefile | ||
430 | @@ -71,7 +71,7 @@ libpthread-routines = nptl-init vars events version pt-interp \ | ||
431 | pthread_rwlockattr_getkind_np \ | ||
432 | pthread_rwlockattr_setkind_np \ | ||
433 | pthread_cond_init pthread_cond_destroy \ | ||
434 | - pthread_cond_wait pthread_cond_timedwait \ | ||
435 | + pthread_cond_wait \ | ||
436 | pthread_cond_signal pthread_cond_broadcast \ | ||
437 | old_pthread_cond_init old_pthread_cond_destroy \ | ||
438 | old_pthread_cond_wait old_pthread_cond_timedwait \ | ||
439 | @@ -181,7 +181,6 @@ CFLAGS-pthread_timedjoin.c = -fexceptions -fasynchronous-unwind-tables | ||
440 | CFLAGS-pthread_once.c = $(uses-callbacks) -fexceptions \ | ||
441 | -fasynchronous-unwind-tables | ||
442 | CFLAGS-pthread_cond_wait.c = -fexceptions -fasynchronous-unwind-tables | ||
443 | -CFLAGS-pthread_cond_timedwait.c = -fexceptions -fasynchronous-unwind-tables | ||
444 | CFLAGS-sem_wait.c = -fexceptions -fasynchronous-unwind-tables | ||
445 | CFLAGS-sem_timedwait.c = -fexceptions -fasynchronous-unwind-tables | ||
446 | |||
447 | @@ -303,8 +302,7 @@ test-xfail-tst-once5 = yes | ||
448 | # Files which must not be linked with libpthread. | ||
449 | tests-nolibpthread = tst-unload | ||
450 | |||
451 | -gen-as-const-headers = pthread-errnos.sym \ | ||
452 | - lowlevelcond.sym lowlevelrwlock.sym \ | ||
453 | +gen-as-const-headers = pthread-errnos.sym lowlevelrwlock.sym \ | ||
454 | unwindbuf.sym \ | ||
455 | lowlevelrobustlock.sym pthread-pi-defines.sym | ||
456 | |||
457 | diff --git a/nptl/lowlevelcond.sym b/nptl/lowlevelcond.sym | ||
458 | deleted file mode 100644 | ||
459 | index 18e1ada..0000000 | ||
460 | --- a/nptl/lowlevelcond.sym | ||
461 | +++ /dev/null | ||
462 | @@ -1,16 +0,0 @@ | ||
463 | -#include <stddef.h> | ||
464 | -#include <sched.h> | ||
465 | -#include <bits/pthreadtypes.h> | ||
466 | -#include <internaltypes.h> | ||
467 | - | ||
468 | --- | ||
469 | - | ||
470 | -cond_lock offsetof (pthread_cond_t, __data.__lock) | ||
471 | -cond_futex offsetof (pthread_cond_t, __data.__futex) | ||
472 | -cond_nwaiters offsetof (pthread_cond_t, __data.__nwaiters) | ||
473 | -total_seq offsetof (pthread_cond_t, __data.__total_seq) | ||
474 | -wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq) | ||
475 | -woken_seq offsetof (pthread_cond_t, __data.__woken_seq) | ||
476 | -dep_mutex offsetof (pthread_cond_t, __data.__mutex) | ||
477 | -broadcast_seq offsetof (pthread_cond_t, __data.__broadcast_seq) | ||
478 | -nwaiters_shift COND_NWAITERS_SHIFT | ||
479 | diff --git a/nptl/nptl-printers.py b/nptl/nptl-printers.py | ||
480 | index e402f23..76adadd 100644 | ||
481 | --- a/nptl/nptl-printers.py | ||
482 | +++ b/nptl/nptl-printers.py | ||
483 | @@ -293,16 +293,6 @@ class MutexAttributesPrinter(object): | ||
484 | elif protocol == PTHREAD_PRIO_PROTECT: | ||
485 | self.values.append(('Protocol', 'Priority protect')) | ||
486 | |||
487 | -CLOCK_IDS = { | ||
488 | - CLOCK_REALTIME: 'CLOCK_REALTIME', | ||
489 | - CLOCK_MONOTONIC: 'CLOCK_MONOTONIC', | ||
490 | - CLOCK_PROCESS_CPUTIME_ID: 'CLOCK_PROCESS_CPUTIME_ID', | ||
491 | - CLOCK_THREAD_CPUTIME_ID: 'CLOCK_THREAD_CPUTIME_ID', | ||
492 | - CLOCK_MONOTONIC_RAW: 'CLOCK_MONOTONIC_RAW', | ||
493 | - CLOCK_REALTIME_COARSE: 'CLOCK_REALTIME_COARSE', | ||
494 | - CLOCK_MONOTONIC_COARSE: 'CLOCK_MONOTONIC_COARSE' | ||
495 | -} | ||
496 | - | ||
497 | class ConditionVariablePrinter(object): | ||
498 | """Pretty printer for pthread_cond_t.""" | ||
499 | |||
500 | @@ -313,24 +303,8 @@ class ConditionVariablePrinter(object): | ||
501 | cond: A gdb.value representing a pthread_cond_t. | ||
502 | """ | ||
503 | |||
504 | - # Since PTHREAD_COND_SHARED is an integer, we need to cast it to void * | ||
505 | - # to be able to compare it to the condvar's __data.__mutex member. | ||
506 | - # | ||
507 | - # While it looks like self.shared_value should be a class variable, | ||
508 | - # that would result in it having an incorrect size if we're loading | ||
509 | - # these printers through .gdbinit for a 64-bit objfile in AMD64. | ||
510 | - # This is because gdb initially assumes the pointer size to be 4 bytes, | ||
511 | - # and only sets it to 8 after loading the 64-bit objfiles. Since | ||
512 | - # .gdbinit runs before any objfiles are loaded, this would effectively | ||
513 | - # make self.shared_value have a size of 4, thus breaking later | ||
514 | - # comparisons with pointers whose types are looked up at runtime. | ||
515 | - void_ptr_type = gdb.lookup_type('void').pointer() | ||
516 | - self.shared_value = gdb.Value(PTHREAD_COND_SHARED).cast(void_ptr_type) | ||
517 | - | ||
518 | data = cond['__data'] | ||
519 | - self.total_seq = data['__total_seq'] | ||
520 | - self.mutex = data['__mutex'] | ||
521 | - self.nwaiters = data['__nwaiters'] | ||
522 | + self.wrefs = data['__wrefs'] | ||
523 | self.values = [] | ||
524 | |||
525 | self.read_values() | ||
526 | @@ -360,7 +334,6 @@ class ConditionVariablePrinter(object): | ||
527 | |||
528 | self.read_status() | ||
529 | self.read_attributes() | ||
530 | - self.read_mutex_info() | ||
531 | |||
532 | def read_status(self): | ||
533 | """Read the status of the condvar. | ||
534 | @@ -369,41 +342,22 @@ class ConditionVariablePrinter(object): | ||
535 | are waiting for it. | ||
536 | """ | ||
537 | |||
538 | - if self.total_seq == PTHREAD_COND_DESTROYED: | ||
539 | - self.values.append(('Status', 'Destroyed')) | ||
540 | - | ||
541 | - self.values.append(('Threads waiting for this condvar', | ||
542 | - self.nwaiters >> COND_NWAITERS_SHIFT)) | ||
543 | + self.values.append(('Threads known to still execute a wait function', | ||
544 | + self.wrefs >> PTHREAD_COND_WREFS_SHIFT)) | ||
545 | |||
546 | def read_attributes(self): | ||
547 | """Read the condvar's attributes.""" | ||
548 | |||
549 | - clock_id = self.nwaiters & ((1 << COND_NWAITERS_SHIFT) - 1) | ||
550 | - | ||
551 | - # clock_id must be casted to int because it's a gdb.Value | ||
552 | - self.values.append(('Clock ID', CLOCK_IDS[int(clock_id)])) | ||
553 | + if (self.wrefs & PTHREAD_COND_CLOCK_MONOTONIC_MASK) != 0: | ||
554 | + self.values.append(('Clock ID', 'CLOCK_MONOTONIC')) | ||
555 | + else: | ||
556 | + self.values.append(('Clock ID', 'CLOCK_REALTIME')) | ||
557 | |||
558 | - shared = (self.mutex == self.shared_value) | ||
559 | - | ||
560 | - if shared: | ||
561 | + if (self.wrefs & PTHREAD_COND_SHARED_MASK) != 0: | ||
562 | self.values.append(('Shared', 'Yes')) | ||
563 | else: | ||
564 | self.values.append(('Shared', 'No')) | ||
565 | |||
566 | - def read_mutex_info(self): | ||
567 | - """Read the data of the mutex this condvar is bound to. | ||
568 | - | ||
569 | - A pthread_cond_t's __data.__mutex member is a void * which | ||
570 | - must be casted to pthread_mutex_t *. For shared condvars, this | ||
571 | - member isn't recorded and has a special value instead. | ||
572 | - """ | ||
573 | - | ||
574 | - if self.mutex and self.mutex != self.shared_value: | ||
575 | - mutex_type = gdb.lookup_type('pthread_mutex_t') | ||
576 | - mutex = self.mutex.cast(mutex_type.pointer()).dereference() | ||
577 | - | ||
578 | - self.values.append(('Mutex', mutex)) | ||
579 | - | ||
580 | class ConditionVariableAttributesPrinter(object): | ||
581 | """Pretty printer for pthread_condattr_t. | ||
582 | |||
583 | @@ -453,10 +407,12 @@ class ConditionVariableAttributesPrinter(object): | ||
584 | created in self.children. | ||
585 | """ | ||
586 | |||
587 | - clock_id = self.condattr & ((1 << COND_NWAITERS_SHIFT) - 1) | ||
588 | + clock_id = (self.condattr >> 1) & ((1 << COND_CLOCK_BITS) - 1) | ||
589 | |||
590 | - # clock_id must be casted to int because it's a gdb.Value | ||
591 | - self.values.append(('Clock ID', CLOCK_IDS[int(clock_id)])) | ||
592 | + if clock_id != 0: | ||
593 | + self.values.append(('Clock ID', 'CLOCK_MONOTONIC')) | ||
594 | + else: | ||
595 | + self.values.append(('Clock ID', 'CLOCK_REALTIME')) | ||
596 | |||
597 | if self.condattr & 1: | ||
598 | self.values.append(('Shared', 'Yes')) | ||
599 | diff --git a/nptl/nptl_lock_constants.pysym b/nptl/nptl_lock_constants.pysym | ||
600 | index 303ec61..2ab3179 100644 | ||
601 | --- a/nptl/nptl_lock_constants.pysym | ||
602 | +++ b/nptl/nptl_lock_constants.pysym | ||
603 | @@ -44,26 +44,13 @@ PTHREAD_PRIO_NONE | ||
604 | PTHREAD_PRIO_INHERIT | ||
605 | PTHREAD_PRIO_PROTECT | ||
606 | |||
607 | --- These values are hardcoded as well: | ||
608 | --- Value of __mutex for shared condvars. | ||
609 | -PTHREAD_COND_SHARED (void *)~0l | ||
610 | - | ||
611 | --- Value of __total_seq for destroyed condvars. | ||
612 | -PTHREAD_COND_DESTROYED -1ull | ||
613 | - | ||
614 | --- __nwaiters encodes the number of threads waiting on a condvar | ||
615 | --- and the clock ID. | ||
616 | --- __nwaiters >> COND_NWAITERS_SHIFT gives us the number of waiters. | ||
617 | -COND_NWAITERS_SHIFT | ||
618 | - | ||
619 | --- Condvar clock IDs | ||
620 | -CLOCK_REALTIME | ||
621 | -CLOCK_MONOTONIC | ||
622 | -CLOCK_PROCESS_CPUTIME_ID | ||
623 | -CLOCK_THREAD_CPUTIME_ID | ||
624 | -CLOCK_MONOTONIC_RAW | ||
625 | -CLOCK_REALTIME_COARSE | ||
626 | -CLOCK_MONOTONIC_COARSE | ||
627 | +-- Condition variable | ||
628 | +-- FIXME Why do macros prefixed with __ cannot be used directly? | ||
629 | +PTHREAD_COND_SHARED_MASK __PTHREAD_COND_SHARED_MASK | ||
630 | +PTHREAD_COND_CLOCK_MONOTONIC_MASK __PTHREAD_COND_CLOCK_MONOTONIC_MASK | ||
631 | +COND_CLOCK_BITS | ||
632 | +-- These values are hardcoded: | ||
633 | +PTHREAD_COND_WREFS_SHIFT 3 | ||
634 | |||
635 | -- Rwlock attributes | ||
636 | PTHREAD_RWLOCK_PREFER_READER_NP | ||
637 | diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h | ||
638 | index 4edc74b..e9992bc 100644 | ||
639 | --- a/nptl/pthreadP.h | ||
640 | +++ b/nptl/pthreadP.h | ||
641 | @@ -167,6 +167,13 @@ enum | ||
642 | #define __PTHREAD_ONCE_FORK_GEN_INCR 4 | ||
643 | |||
644 | |||
645 | +/* Condition variable definitions. See __pthread_cond_wait_common. | ||
646 | + Need to be defined here so there is one place from which | ||
647 | + nptl_lock_constants can grab them. */ | ||
648 | +#define __PTHREAD_COND_CLOCK_MONOTONIC_MASK 2 | ||
649 | +#define __PTHREAD_COND_SHARED_MASK 1 | ||
650 | + | ||
651 | + | ||
652 | /* Internal variables. */ | ||
653 | |||
654 | |||
655 | diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c | ||
656 | index 552fd42..87c0755 100644 | ||
657 | --- a/nptl/pthread_cond_broadcast.c | ||
658 | +++ b/nptl/pthread_cond_broadcast.c | ||
659 | @@ -19,72 +19,71 @@ | ||
660 | #include <endian.h> | ||
661 | #include <errno.h> | ||
662 | #include <sysdep.h> | ||
663 | -#include <lowlevellock.h> | ||
664 | +#include <futex-internal.h> | ||
665 | #include <pthread.h> | ||
666 | #include <pthreadP.h> | ||
667 | #include <stap-probe.h> | ||
668 | +#include <atomic.h> | ||
669 | |||
670 | #include <shlib-compat.h> | ||
671 | -#include <kernel-features.h> | ||
672 | |||
673 | +#include "pthread_cond_common.c" | ||
674 | |||
675 | + | ||
676 | +/* We do the following steps from __pthread_cond_signal in one critical | ||
677 | + section: (1) signal all waiters in G1, (2) close G1 so that it can become | ||
678 | + the new G2 and make G2 the new G1, and (3) signal all waiters in the new | ||
679 | + G1. We don't need to do all these steps if there are no waiters in G1 | ||
680 | + and/or G2. See __pthread_cond_signal for further details. */ | ||
681 | int | ||
682 | __pthread_cond_broadcast (pthread_cond_t *cond) | ||
683 | { | ||
684 | LIBC_PROBE (cond_broadcast, 1, cond); | ||
685 | |||
686 | - int pshared = (cond->__data.__mutex == (void *) ~0l) | ||
687 | - ? LLL_SHARED : LLL_PRIVATE; | ||
688 | - /* Make sure we are alone. */ | ||
689 | - lll_lock (cond->__data.__lock, pshared); | ||
690 | + unsigned int wrefs = atomic_load_relaxed (&cond->__data.__wrefs); | ||
691 | + if (wrefs >> 3 == 0) | ||
692 | + return 0; | ||
693 | + int private = __condvar_get_private (wrefs); | ||
694 | + | ||
695 | + __condvar_acquire_lock (cond, private); | ||
696 | |||
697 | - /* Are there any waiters to be woken? */ | ||
698 | - if (cond->__data.__total_seq > cond->__data.__wakeup_seq) | ||
699 | + unsigned long long int wseq = __condvar_load_wseq_relaxed (cond); | ||
700 | + unsigned int g2 = wseq & 1; | ||
701 | + unsigned int g1 = g2 ^ 1; | ||
702 | + wseq >>= 1; | ||
703 | + bool do_futex_wake = false; | ||
704 | + | ||
705 | + /* Step (1): signal all waiters remaining in G1. */ | ||
706 | + if (cond->__data.__g_size[g1] != 0) | ||
707 | { | ||
708 | - /* Yes. Mark them all as woken. */ | ||
709 | - cond->__data.__wakeup_seq = cond->__data.__total_seq; | ||
710 | - cond->__data.__woken_seq = cond->__data.__total_seq; | ||
711 | - cond->__data.__futex = (unsigned int) cond->__data.__total_seq * 2; | ||
712 | - int futex_val = cond->__data.__futex; | ||
713 | - /* Signal that a broadcast happened. */ | ||
714 | - ++cond->__data.__broadcast_seq; | ||
715 | - | ||
716 | - /* We are done. */ | ||
717 | - lll_unlock (cond->__data.__lock, pshared); | ||
718 | - | ||
719 | - /* Wake everybody. */ | ||
720 | - pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; | ||
721 | - | ||
722 | - /* Do not use requeue for pshared condvars. */ | ||
723 | - if (mut == (void *) ~0l | ||
724 | - || PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT) | ||
725 | - goto wake_all; | ||
726 | - | ||
727 | -#if (defined lll_futex_cmp_requeue_pi \ | ||
728 | - && defined __ASSUME_REQUEUE_PI) | ||
729 | - if (USE_REQUEUE_PI (mut)) | ||
730 | - { | ||
731 | - if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX, | ||
732 | - &mut->__data.__lock, futex_val, | ||
733 | - LLL_PRIVATE) == 0) | ||
734 | - return 0; | ||
735 | - } | ||
736 | - else | ||
737 | -#endif | ||
738 | - /* lll_futex_requeue returns 0 for success and non-zero | ||
739 | - for errors. */ | ||
740 | - if (!__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1, | ||
741 | - INT_MAX, &mut->__data.__lock, | ||
742 | - futex_val, LLL_PRIVATE), 0)) | ||
743 | - return 0; | ||
744 | - | ||
745 | -wake_all: | ||
746 | - lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared); | ||
747 | - return 0; | ||
748 | + /* Add as many signals as the remaining size of the group. */ | ||
749 | + atomic_fetch_add_relaxed (cond->__data.__g_signals + g1, | ||
750 | + cond->__data.__g_size[g1] << 1); | ||
751 | + cond->__data.__g_size[g1] = 0; | ||
752 | + | ||
753 | + /* We need to wake G1 waiters before we quiesce G1 below. */ | ||
754 | + /* TODO Only set it if there are indeed futex waiters. We could | ||
755 | + also try to move this out of the critical section in cases when | ||
756 | + G2 is empty (and we don't need to quiesce). */ | ||
757 | + futex_wake (cond->__data.__g_signals + g1, INT_MAX, private); | ||
758 | } | ||
759 | |||
760 | - /* We are done. */ | ||
761 | - lll_unlock (cond->__data.__lock, pshared); | ||
762 | + /* G1 is complete. Step (2) is next unless there are no waiters in G2, in | ||
763 | + which case we can stop. */ | ||
764 | + if (__condvar_quiesce_and_switch_g1 (cond, wseq, &g1, private)) | ||
765 | + { | ||
766 | + /* Step (3): Send signals to all waiters in the old G2 / new G1. */ | ||
767 | + atomic_fetch_add_relaxed (cond->__data.__g_signals + g1, | ||
768 | + cond->__data.__g_size[g1] << 1); | ||
769 | + cond->__data.__g_size[g1] = 0; | ||
770 | + /* TODO Only set it if there are indeed futex waiters. */ | ||
771 | + do_futex_wake = true; | ||
772 | + } | ||
773 | + | ||
774 | + __condvar_release_lock (cond, private); | ||
775 | + | ||
776 | + if (do_futex_wake) | ||
777 | + futex_wake (cond->__data.__g_signals + g1, INT_MAX, private); | ||
778 | |||
779 | return 0; | ||
780 | } | ||
781 | diff --git a/nptl/pthread_cond_common.c b/nptl/pthread_cond_common.c | ||
782 | new file mode 100644 | ||
783 | index 0000000..b374396 | ||
784 | --- /dev/null | ||
785 | +++ b/nptl/pthread_cond_common.c | ||
786 | @@ -0,0 +1,466 @@ | ||
787 | +/* pthread_cond_common -- shared code for condition variable. | ||
788 | + Copyright (C) 2016 Free Software Foundation, Inc. | ||
789 | + This file is part of the GNU C Library. | ||
790 | + | ||
791 | + The GNU C Library is free software; you can redistribute it and/or | ||
792 | + modify it under the terms of the GNU Lesser General Public | ||
793 | + License as published by the Free Software Foundation; either | ||
794 | + version 2.1 of the License, or (at your option) any later version. | ||
795 | + | ||
796 | + The GNU C Library is distributed in the hope that it will be useful, | ||
797 | + but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
798 | + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
799 | + Lesser General Public License for more details. | ||
800 | + | ||
801 | + You should have received a copy of the GNU Lesser General Public | ||
802 | + License along with the GNU C Library; if not, see | ||
803 | + <http://www.gnu.org/licenses/>. */ | ||
804 | + | ||
805 | +#include <atomic.h> | ||
806 | +#include <stdint.h> | ||
807 | +#include <pthread.h> | ||
808 | +#include <libc-internal.h> | ||
809 | + | ||
810 | +/* We need 3 least-significant bits on __wrefs for something else. */ | ||
811 | +#define __PTHREAD_COND_MAX_GROUP_SIZE ((unsigned) 1 << 29) | ||
812 | + | ||
813 | +#if __HAVE_64B_ATOMICS == 1 | ||
814 | + | ||
815 | +static uint64_t __attribute__ ((unused)) | ||
816 | +__condvar_load_wseq_relaxed (pthread_cond_t *cond) | ||
817 | +{ | ||
818 | + return atomic_load_relaxed (&cond->__data.__wseq); | ||
819 | +} | ||
820 | + | ||
821 | +static uint64_t __attribute__ ((unused)) | ||
822 | +__condvar_fetch_add_wseq_acquire (pthread_cond_t *cond, unsigned int val) | ||
823 | +{ | ||
824 | + return atomic_fetch_add_acquire (&cond->__data.__wseq, val); | ||
825 | +} | ||
826 | + | ||
827 | +static uint64_t __attribute__ ((unused)) | ||
828 | +__condvar_fetch_xor_wseq_release (pthread_cond_t *cond, unsigned int val) | ||
829 | +{ | ||
830 | + return atomic_fetch_xor_release (&cond->__data.__wseq, val); | ||
831 | +} | ||
832 | + | ||
833 | +static uint64_t __attribute__ ((unused)) | ||
834 | +__condvar_load_g1_start_relaxed (pthread_cond_t *cond) | ||
835 | +{ | ||
836 | + return atomic_load_relaxed (&cond->__data.__g1_start); | ||
837 | +} | ||
838 | + | ||
839 | +static void __attribute__ ((unused)) | ||
840 | +__condvar_add_g1_start_relaxed (pthread_cond_t *cond, unsigned int val) | ||
841 | +{ | ||
842 | + atomic_store_relaxed (&cond->__data.__g1_start, | ||
843 | + atomic_load_relaxed (&cond->__data.__g1_start) + val); | ||
844 | +} | ||
845 | + | ||
846 | +#else | ||
847 | + | ||
848 | +/* We use two 64b counters: __wseq and __g1_start. They are monotonically | ||
849 | + increasing and single-writer-multiple-readers counters, so we can implement | ||
850 | + load, fetch-and-add, and fetch-and-xor operations even when we just have | ||
851 | + 32b atomics. Values we add or xor are less than or equal to 1<<31 (*), | ||
852 | + so we only have to make overflow-and-addition atomic wrt. to concurrent | ||
853 | + load operations and xor operations. To do that, we split each counter into | ||
854 | + two 32b values of which we reserve the MSB of each to represent an | ||
855 | + overflow from the lower-order half to the higher-order half. | ||
856 | + | ||
857 | + In the common case, the state is (higher-order / lower-order half, and . is | ||
858 | + basically concatenation of the bits): | ||
859 | + 0.h / 0.l = h.l | ||
860 | + | ||
861 | + When we add a value of x that overflows (i.e., 0.l + x == 1.L), we run the | ||
862 | + following steps S1-S4 (the values these represent are on the right-hand | ||
863 | + side): | ||
864 | + S1: 0.h / 1.L == (h+1).L | ||
865 | + S2: 1.(h+1) / 1.L == (h+1).L | ||
866 | + S3: 1.(h+1) / 0.L == (h+1).L | ||
867 | + S4: 0.(h+1) / 0.L == (h+1).L | ||
868 | + If the LSB of the higher-order half is set, readers will ignore the | ||
869 | + overflow bit in the lower-order half. | ||
870 | + | ||
871 | + To get an atomic snapshot in load operations, we exploit that the | ||
872 | + higher-order half is monotonically increasing; if we load a value V from | ||
873 | + it, then read the lower-order half, and then read the higher-order half | ||
874 | + again and see the same value V, we know that both halves have existed in | ||
875 | + the sequence of values the full counter had. This is similar to the | ||
876 | + validated reads in the time-based STMs in GCC's libitm (e.g., | ||
877 | + method_ml_wt). | ||
878 | + | ||
879 | + The xor operation needs to be an atomic read-modify-write. The write | ||
880 | + itself is not an issue as it affects just the lower-order half but not bits | ||
881 | + used in the add operation. To make the full fetch-and-xor atomic, we | ||
882 | + exploit that concurrently, the value can increase by at most 1<<31 (*): The | ||
883 | + xor operation is only called while having acquired the lock, so not more | ||
884 | + than __PTHREAD_COND_MAX_GROUP_SIZE waiters can enter concurrently and thus | ||
885 | + increment __wseq. Therefore, if the xor operation observes a value of | ||
886 | + __wseq, then the value it applies the modification to later on can be | ||
887 | + derived (see below). | ||
888 | + | ||
889 | + One benefit of this scheme is that this makes load operations | ||
890 | + obstruction-free because unlike if we would just lock the counter, readers | ||
891 | + can almost always interpret a snapshot of each halves. Readers can be | ||
892 | + forced to read a new snapshot when the read is concurrent with an overflow. | ||
893 | + However, overflows will happen infrequently, so load operations are | ||
894 | + practically lock-free. | ||
895 | + | ||
896 | + (*) The highest value we add is __PTHREAD_COND_MAX_GROUP_SIZE << 2 to | ||
897 | + __g1_start (the two extra bits are for the lock in the two LSBs of | ||
898 | + __g1_start). */ | ||
899 | + | ||
900 | +typedef struct | ||
901 | +{ | ||
902 | + unsigned int low; | ||
903 | + unsigned int high; | ||
904 | +} _condvar_lohi; | ||
905 | + | ||
906 | +static uint64_t | ||
907 | +__condvar_fetch_add_64_relaxed (_condvar_lohi *lh, unsigned int op) | ||
908 | +{ | ||
909 | + /* S1. Note that this is an atomic read-modify-write so it extends the | ||
910 | + release sequence of release MO store at S3. */ | ||
911 | + unsigned int l = atomic_fetch_add_relaxed (&lh->low, op); | ||
912 | + unsigned int h = atomic_load_relaxed (&lh->high); | ||
913 | + uint64_t result = ((uint64_t) h << 31) | l; | ||
914 | + l += op; | ||
915 | + if ((l >> 31) > 0) | ||
916 | + { | ||
917 | + /* Overflow. Need to increment higher-order half. Note that all | ||
918 | + add operations are ordered in happens-before. */ | ||
919 | + h++; | ||
920 | + /* S2. Release MO to synchronize with the loads of the higher-order half | ||
921 | + in the load operation. See __condvar_load_64_relaxed. */ | ||
922 | + atomic_store_release (&lh->high, h | ((unsigned int) 1 << 31)); | ||
923 | + l ^= (unsigned int) 1 << 31; | ||
924 | + /* S3. See __condvar_load_64_relaxed. */ | ||
925 | + atomic_store_release (&lh->low, l); | ||
926 | + /* S4. Likewise. */ | ||
927 | + atomic_store_release (&lh->high, h); | ||
928 | + } | ||
929 | + return result; | ||
930 | +} | ||
931 | + | ||
932 | +static uint64_t | ||
933 | +__condvar_load_64_relaxed (_condvar_lohi *lh) | ||
934 | +{ | ||
935 | + unsigned int h, l, h2; | ||
936 | + do | ||
937 | + { | ||
938 | + /* This load and the second one below to the same location read from the | ||
939 | + stores in the overflow handling of the add operation or the | ||
940 | + initializing stores (which is a simple special case because | ||
941 | + initialization always completely happens before further use). | ||
942 | + Because no two stores to the higher-order half write the same value, | ||
943 | + the loop ensures that if we continue to use the snapshot, this load | ||
944 | + and the second one read from the same store operation. All candidate | ||
945 | + store operations have release MO. | ||
946 | + If we read from S2 in the first load, then we will see the value of | ||
947 | + S1 on the next load (because we synchronize with S2), or a value | ||
948 | + later in modification order. We correctly ignore the lower-half's | ||
949 | + overflow bit in this case. If we read from S4, then we will see the | ||
950 | + value of S3 in the next load (or a later value), which does not have | ||
951 | + the overflow bit set anymore. | ||
952 | + */ | ||
953 | + h = atomic_load_acquire (&lh->high); | ||
954 | + /* This will read from the release sequence of S3 (i.e, either the S3 | ||
955 | + store or the read-modify-writes at S1 following S3 in modification | ||
956 | + order). Thus, the read synchronizes with S3, and the following load | ||
957 | + of the higher-order half will read from the matching S2 (or a later | ||
958 | + value). | ||
959 | + Thus, if we read a lower-half value here that already overflowed and | ||
960 | + belongs to an increased higher-order half value, we will see the | ||
961 | + latter and h and h2 will not be equal. */ | ||
962 | + l = atomic_load_acquire (&lh->low); | ||
963 | + /* See above. */ | ||
964 | + h2 = atomic_load_relaxed (&lh->high); | ||
965 | + } | ||
966 | + while (h != h2); | ||
967 | + if (((l >> 31) > 0) && ((h >> 31) > 0)) | ||
968 | + l ^= (unsigned int) 1 << 31; | ||
969 | + return ((uint64_t) (h & ~((unsigned int) 1 << 31)) << 31) + l; | ||
970 | +} | ||
971 | + | ||
972 | +static uint64_t __attribute__ ((unused)) | ||
973 | +__condvar_load_wseq_relaxed (pthread_cond_t *cond) | ||
974 | +{ | ||
975 | + return __condvar_load_64_relaxed ((_condvar_lohi *) &cond->__data.__wseq32); | ||
976 | +} | ||
977 | + | ||
978 | +static uint64_t __attribute__ ((unused)) | ||
979 | +__condvar_fetch_add_wseq_acquire (pthread_cond_t *cond, unsigned int val) | ||
980 | +{ | ||
981 | + uint64_t r = __condvar_fetch_add_64_relaxed | ||
982 | + ((_condvar_lohi *) &cond->__data.__wseq32, val); | ||
983 | + atomic_thread_fence_acquire (); | ||
984 | + return r; | ||
985 | +} | ||
986 | + | ||
987 | +static uint64_t __attribute__ ((unused)) | ||
988 | +__condvar_fetch_xor_wseq_release (pthread_cond_t *cond, unsigned int val) | ||
989 | +{ | ||
990 | + _condvar_lohi *lh = (_condvar_lohi *) &cond->__data.__wseq32; | ||
991 | + /* First, get the current value. See __condvar_load_64_relaxed. */ | ||
992 | + unsigned int h, l, h2; | ||
993 | + do | ||
994 | + { | ||
995 | + h = atomic_load_acquire (&lh->high); | ||
996 | + l = atomic_load_acquire (&lh->low); | ||
997 | + h2 = atomic_load_relaxed (&lh->high); | ||
998 | + } | ||
999 | + while (h != h2); | ||
1000 | + if (((l >> 31) > 0) && ((h >> 31) == 0)) | ||
1001 | + h++; | ||
1002 | + h &= ~((unsigned int) 1 << 31); | ||
1003 | + l &= ~((unsigned int) 1 << 31); | ||
1004 | + | ||
1005 | + /* Now modify. Due to the coherence rules, the prior load will read a value | ||
1006 | + earlier in modification order than the following fetch-xor. | ||
1007 | + This uses release MO to make the full operation have release semantics | ||
1008 | + (all other operations access the lower-order half). */ | ||
1009 | + unsigned int l2 = atomic_fetch_xor_release (&lh->low, val) | ||
1010 | + & ~((unsigned int) 1 << 31); | ||
1011 | + if (l2 < l) | ||
1012 | + /* The lower-order half overflowed in the meantime. This happened exactly | ||
1013 | + once due to the limit on concurrent waiters (see above). */ | ||
1014 | + h++; | ||
1015 | + return ((uint64_t) h << 31) + l2; | ||
1016 | +} | ||
1017 | + | ||
1018 | +static uint64_t __attribute__ ((unused)) | ||
1019 | +__condvar_load_g1_start_relaxed (pthread_cond_t *cond) | ||
1020 | +{ | ||
1021 | + return __condvar_load_64_relaxed | ||
1022 | + ((_condvar_lohi *) &cond->__data.__g1_start32); | ||
1023 | +} | ||
1024 | + | ||
1025 | +static void __attribute__ ((unused)) | ||
1026 | +__condvar_add_g1_start_relaxed (pthread_cond_t *cond, unsigned int val) | ||
1027 | +{ | ||
1028 | + ignore_value (__condvar_fetch_add_64_relaxed | ||
1029 | + ((_condvar_lohi *) &cond->__data.__g1_start32, val)); | ||
1030 | +} | ||
1031 | + | ||
1032 | +#endif /* !__HAVE_64B_ATOMICS */ | ||
1033 | + | ||
1034 | + | ||
1035 | +/* The lock that signalers use. See pthread_cond_wait_common for uses. | ||
1036 | + The lock is our normal three-state lock: not acquired (0) / acquired (1) / | ||
1037 | + acquired-with-futex_wake-request (2). However, we need to preserve the | ||
1038 | + other bits in the unsigned int used for the lock, and therefore it is a | ||
1039 | + little more complex. */ | ||
1040 | +static void __attribute__ ((unused)) | ||
1041 | +__condvar_acquire_lock (pthread_cond_t *cond, int private) | ||
1042 | +{ | ||
1043 | + unsigned int s = atomic_load_relaxed (&cond->__data.__g1_orig_size); | ||
1044 | + while ((s & 3) == 0) | ||
1045 | + { | ||
1046 | + if (atomic_compare_exchange_weak_acquire (&cond->__data.__g1_orig_size, | ||
1047 | + &s, s | 1)) | ||
1048 | + return; | ||
1049 | + /* TODO Spinning and back-off. */ | ||
1050 | + } | ||
1051 | + /* We can't change from not acquired to acquired, so try to change to | ||
1052 | + acquired-with-futex-wake-request and do a futex wait if we cannot change | ||
1053 | + from not acquired. */ | ||
1054 | + while (1) | ||
1055 | + { | ||
1056 | + while ((s & 3) != 2) | ||
1057 | + { | ||
1058 | + if (atomic_compare_exchange_weak_acquire | ||
1059 | + (&cond->__data.__g1_orig_size, &s, (s & ~(unsigned int) 3) | 2)) | ||
1060 | + { | ||
1061 | + if ((s & 3) == 0) | ||
1062 | + return; | ||
1063 | + break; | ||
1064 | + } | ||
1065 | + /* TODO Back off. */ | ||
1066 | + } | ||
1067 | + futex_wait_simple (&cond->__data.__g1_orig_size, | ||
1068 | + (s & ~(unsigned int) 3) | 2, private); | ||
1069 | + /* Reload so we see a recent value. */ | ||
1070 | + s = atomic_load_relaxed (&cond->__data.__g1_orig_size); | ||
1071 | + } | ||
1072 | +} | ||
1073 | + | ||
1074 | +/* See __condvar_acquire_lock. */ | ||
1075 | +static void __attribute__ ((unused)) | ||
1076 | +__condvar_release_lock (pthread_cond_t *cond, int private) | ||
1077 | +{ | ||
1078 | + if ((atomic_fetch_and_release (&cond->__data.__g1_orig_size, | ||
1079 | + ~(unsigned int) 3) & 3) | ||
1080 | + == 2) | ||
1081 | + futex_wake (&cond->__data.__g1_orig_size, 1, private); | ||
1082 | +} | ||
1083 | + | ||
1084 | +/* Only use this when having acquired the lock. */ | ||
1085 | +static unsigned int __attribute__ ((unused)) | ||
1086 | +__condvar_get_orig_size (pthread_cond_t *cond) | ||
1087 | +{ | ||
1088 | + return atomic_load_relaxed (&cond->__data.__g1_orig_size) >> 2; | ||
1089 | +} | ||
1090 | + | ||
1091 | +/* Only use this when having acquired the lock. */ | ||
1092 | +static void __attribute__ ((unused)) | ||
1093 | +__condvar_set_orig_size (pthread_cond_t *cond, unsigned int size) | ||
1094 | +{ | ||
1095 | + /* We have acquired the lock, but might get one concurrent update due to a | ||
1096 | + lock state change from acquired to acquired-with-futex_wake-request. | ||
1097 | + The store with relaxed MO is fine because there will be no further | ||
1098 | + changes to the lock bits nor the size, and we will subsequently release | ||
1099 | + the lock with release MO. */ | ||
1100 | + unsigned int s; | ||
1101 | + s = (atomic_load_relaxed (&cond->__data.__g1_orig_size) & 3) | ||
1102 | + | (size << 2); | ||
1103 | + if ((atomic_exchange_relaxed (&cond->__data.__g1_orig_size, s) & 3) | ||
1104 | + != (s & 3)) | ||
1105 | + atomic_store_relaxed (&cond->__data.__g1_orig_size, (size << 2) | 2); | ||
1106 | +} | ||
1107 | + | ||
1108 | +/* Returns FUTEX_SHARED or FUTEX_PRIVATE based on the provided __wrefs | ||
1109 | + value. */ | ||
1110 | +static int __attribute__ ((unused)) | ||
1111 | +__condvar_get_private (int flags) | ||
1112 | +{ | ||
1113 | + if ((flags & __PTHREAD_COND_SHARED_MASK) == 0) | ||
1114 | + return FUTEX_PRIVATE; | ||
1115 | + else | ||
1116 | + return FUTEX_SHARED; | ||
1117 | +} | ||
1118 | + | ||
1119 | +/* This closes G1 (whose index is in G1INDEX), waits for all futex waiters to | ||
1120 | + leave G1, converts G1 into a fresh G2, and then switches group roles so that | ||
1121 | + the former G2 becomes the new G1 ending at the current __wseq value when we | ||
1122 | + eventually make the switch (WSEQ is just an observation of __wseq by the | ||
1123 | + signaler). | ||
1124 | + If G2 is empty, it will not switch groups because then it would create an | ||
1125 | + empty G1 which would require switching groups again on the next signal. | ||
1126 | + Returns false iff groups were not switched because G2 was empty. */ | ||
1127 | +static bool __attribute__ ((unused)) | ||
1128 | +__condvar_quiesce_and_switch_g1 (pthread_cond_t *cond, uint64_t wseq, | ||
1129 | + unsigned int *g1index, int private) | ||
1130 | +{ | ||
1131 | + const unsigned int maxspin = 0; | ||
1132 | + unsigned int g1 = *g1index; | ||
1133 | + | ||
1134 | + /* If there is no waiter in G2, we don't do anything. The expression may | ||
1135 | + look odd but remember that __g_size might hold a negative value, so | ||
1136 | + putting the expression this way avoids relying on implementation-defined | ||
1137 | + behavior. | ||
1138 | + Note that this works correctly for a zero-initialized condvar too. */ | ||
1139 | + unsigned int old_orig_size = __condvar_get_orig_size (cond); | ||
1140 | + uint64_t old_g1_start = __condvar_load_g1_start_relaxed (cond) >> 1; | ||
1141 | + if (((unsigned) (wseq - old_g1_start - old_orig_size) | ||
1142 | + + cond->__data.__g_size[g1 ^ 1]) == 0) | ||
1143 | + return false; | ||
1144 | + | ||
1145 | + /* Now try to close and quiesce G1. We have to consider the following kinds | ||
1146 | + of waiters: | ||
1147 | + * Waiters from less recent groups than G1 are not affected because | ||
1148 | + nothing will change for them apart from __g1_start getting larger. | ||
1149 | + * New waiters arriving concurrently with the group switching will all go | ||
1150 | + into G2 until we atomically make the switch. Waiters existing in G2 | ||
1151 | + are not affected. | ||
1152 | + * Waiters in G1 will be closed out immediately by setting a flag in | ||
1153 | + __g_signals, which will prevent waiters from blocking using a futex on | ||
1154 | + __g_signals and also notifies them that the group is closed. As a | ||
1155 | + result, they will eventually remove their group reference, allowing us | ||
1156 | + to close switch group roles. */ | ||
1157 | + | ||
1158 | + /* First, set the closed flag on __g_signals. This tells waiters that are | ||
1159 | + about to wait that they shouldn't do that anymore. This basically | ||
1160 | + serves as an advance notificaton of the upcoming change to __g1_start; | ||
1161 | + waiters interpret it as if __g1_start was larger than their waiter | ||
1162 | + sequence position. This allows us to change __g1_start after waiting | ||
1163 | + for all existing waiters with group references to leave, which in turn | ||
1164 | + makes recovery after stealing a signal simpler because it then can be | ||
1165 | + skipped if __g1_start indicates that the group is closed (otherwise, | ||
1166 | + we would have to recover always because waiters don't know how big their | ||
1167 | + groups are). Relaxed MO is fine. */ | ||
1168 | + atomic_fetch_or_relaxed (cond->__data.__g_signals + g1, 1); | ||
1169 | + | ||
1170 | + /* Wait until there are no group references anymore. The fetch-or operation | ||
1171 | + injects us into the modification order of __g_refs; release MO ensures | ||
1172 | + that waiters incrementing __g_refs after our fetch-or see the previous | ||
1173 | + changes to __g_signals and to __g1_start that had to happen before we can | ||
1174 | + switch this G1 and alias with an older group (we have two groups, so | ||
1175 | + aliasing requires switching group roles twice). Note that nobody else | ||
1176 | + can have set the wake-request flag, so we do not have to act upon it. | ||
1177 | + | ||
1178 | + Also note that it is harmless if older waiters or waiters from this G1 | ||
1179 | + get a group reference after we have quiesced the group because it will | ||
1180 | + remain closed for them either because of the closed flag in __g_signals | ||
1181 | + or the later update to __g1_start. New waiters will never arrive here | ||
1182 | + but instead continue to go into the still current G2. */ | ||
1183 | + unsigned r = atomic_fetch_or_release (cond->__data.__g_refs + g1, 0); | ||
1184 | + while ((r >> 1) > 0) | ||
1185 | + { | ||
1186 | + for (unsigned int spin = maxspin; ((r >> 1) > 0) && (spin > 0); spin--) | ||
1187 | + { | ||
1188 | + /* TODO Back off. */ | ||
1189 | + r = atomic_load_relaxed (cond->__data.__g_refs + g1); | ||
1190 | + } | ||
1191 | + if ((r >> 1) > 0) | ||
1192 | + { | ||
1193 | + /* There is still a waiter after spinning. Set the wake-request | ||
1194 | + flag and block. Relaxed MO is fine because this is just about | ||
1195 | + this futex word. */ | ||
1196 | + r = atomic_fetch_or_relaxed (cond->__data.__g_refs + g1, 1); | ||
1197 | + | ||
1198 | + if ((r >> 1) > 0) | ||
1199 | + futex_wait_simple (cond->__data.__g_refs + g1, r, private); | ||
1200 | + /* Reload here so we eventually see the most recent value even if we | ||
1201 | + do not spin. */ | ||
1202 | + r = atomic_load_relaxed (cond->__data.__g_refs + g1); | ||
1203 | + } | ||
1204 | + } | ||
1205 | + /* Acquire MO so that we synchronize with the release operation that waiters | ||
1206 | + use to decrement __g_refs and thus happen after the waiters we waited | ||
1207 | + for. */ | ||
1208 | + atomic_thread_fence_acquire (); | ||
1209 | + | ||
1210 | + /* Update __g1_start, which finishes closing this group. The value we add | ||
1211 | + will never be negative because old_orig_size can only be zero when we | ||
1212 | + switch groups the first time after a condvar was initialized, in which | ||
1213 | + case G1 will be at index 1 and we will add a value of 1. See above for | ||
1214 | + why this takes place after waiting for quiescence of the group. | ||
1215 | + Relaxed MO is fine because the change comes with no additional | ||
1216 | + constraints that others would have to observe. */ | ||
1217 | + __condvar_add_g1_start_relaxed (cond, | ||
1218 | + (old_orig_size << 1) + (g1 == 1 ? 1 : - 1)); | ||
1219 | + | ||
1220 | + /* Now reopen the group, thus enabling waiters to again block using the | ||
1221 | + futex controlled by __g_signals. Release MO so that observers that see | ||
1222 | + no signals (and thus can block) also see the write __g1_start and thus | ||
1223 | + that this is now a new group (see __pthread_cond_wait_common for the | ||
1224 | + matching acquire MO loads). */ | ||
1225 | + atomic_store_release (cond->__data.__g_signals + g1, 0); | ||
1226 | + | ||
1227 | + /* At this point, the old G1 is now a valid new G2 (but not in use yet). | ||
1228 | + No old waiter can neither grab a signal nor acquire a reference without | ||
1229 | + noticing that __g1_start is larger. | ||
1230 | + We can now publish the group switch by flipping the G2 index in __wseq. | ||
1231 | + Release MO so that this synchronizes with the acquire MO operation | ||
1232 | + waiters use to obtain a position in the waiter sequence. */ | ||
1233 | + wseq = __condvar_fetch_xor_wseq_release (cond, 1) >> 1; | ||
1234 | + g1 ^= 1; | ||
1235 | + *g1index ^= 1; | ||
1236 | + | ||
1237 | + /* These values are just observed by signalers, and thus protected by the | ||
1238 | + lock. */ | ||
1239 | + unsigned int orig_size = wseq - (old_g1_start + old_orig_size); | ||
1240 | + __condvar_set_orig_size (cond, orig_size); | ||
1241 | + /* Use and addition to not loose track of cancellations in what was | ||
1242 | + previously G2. */ | ||
1243 | + cond->__data.__g_size[g1] += orig_size; | ||
1244 | + | ||
1245 | + /* The new G1's size may be zero because of cancellations during its time | ||
1246 | + as G2. If this happens, there are no waiters that have to receive a | ||
1247 | + signal, so we do not need to add any and return false. */ | ||
1248 | + if (cond->__data.__g_size[g1] == 0) | ||
1249 | + return false; | ||
1250 | + | ||
1251 | + return true; | ||
1252 | +} | ||
1253 | diff --git a/nptl/pthread_cond_destroy.c b/nptl/pthread_cond_destroy.c | ||
1254 | index 1acd804..5845c6a 100644 | ||
1255 | --- a/nptl/pthread_cond_destroy.c | ||
1256 | +++ b/nptl/pthread_cond_destroy.c | ||
1257 | @@ -20,66 +20,42 @@ | ||
1258 | #include <shlib-compat.h> | ||
1259 | #include "pthreadP.h" | ||
1260 | #include <stap-probe.h> | ||
1261 | - | ||
1262 | - | ||
1263 | +#include <atomic.h> | ||
1264 | +#include <futex-internal.h> | ||
1265 | + | ||
1266 | +#include "pthread_cond_common.c" | ||
1267 | + | ||
1268 | +/* See __pthread_cond_wait for a high-level description of the algorithm. | ||
1269 | + | ||
1270 | + A correct program must make sure that no waiters are blocked on the condvar | ||
1271 | + when it is destroyed, and that there are no concurrent signals or | ||
1272 | + broadcasts. To wake waiters reliably, the program must signal or | ||
1273 | + broadcast while holding the mutex or after having held the mutex. It must | ||
1274 | + also ensure that no signal or broadcast are still pending to unblock | ||
1275 | + waiters; IOW, because waiters can wake up spuriously, the program must | ||
1276 | + effectively ensure that destruction happens after the execution of those | ||
1277 | + signal or broadcast calls. | ||
1278 | + Thus, we can assume that all waiters that are still accessing the condvar | ||
1279 | + have been woken. We wait until they have confirmed to have woken up by | ||
1280 | + decrementing __wrefs. */ | ||
1281 | int | ||
1282 | __pthread_cond_destroy (pthread_cond_t *cond) | ||
1283 | { | ||
1284 | - int pshared = (cond->__data.__mutex == (void *) ~0l) | ||
1285 | - ? LLL_SHARED : LLL_PRIVATE; | ||
1286 | - | ||
1287 | LIBC_PROBE (cond_destroy, 1, cond); | ||
1288 | |||
1289 | - /* Make sure we are alone. */ | ||
1290 | - lll_lock (cond->__data.__lock, pshared); | ||
1291 | - | ||
1292 | - if (cond->__data.__total_seq > cond->__data.__wakeup_seq) | ||
1293 | - { | ||
1294 | - /* If there are still some waiters which have not been | ||
1295 | - woken up, this is an application bug. */ | ||
1296 | - lll_unlock (cond->__data.__lock, pshared); | ||
1297 | - return EBUSY; | ||
1298 | - } | ||
1299 | - | ||
1300 | - /* Tell pthread_cond_*wait that this condvar is being destroyed. */ | ||
1301 | - cond->__data.__total_seq = -1ULL; | ||
1302 | - | ||
1303 | - /* If there are waiters which have been already signalled or | ||
1304 | - broadcasted, but still are using the pthread_cond_t structure, | ||
1305 | - pthread_cond_destroy needs to wait for them. */ | ||
1306 | - unsigned int nwaiters = cond->__data.__nwaiters; | ||
1307 | - | ||
1308 | - if (nwaiters >= (1 << COND_NWAITERS_SHIFT)) | ||
1309 | + /* Set the wake request flag. We could also spin, but destruction that is | ||
1310 | + concurrent with still-active waiters is probably neither common nor | ||
1311 | + performance critical. Acquire MO to synchronize with waiters confirming | ||
1312 | + that they finished. */ | ||
1313 | + unsigned int wrefs = atomic_fetch_or_acquire (&cond->__data.__wrefs, 4); | ||
1314 | + int private = __condvar_get_private (wrefs); | ||
1315 | + while (wrefs >> 3 != 0) | ||
1316 | { | ||
1317 | - /* Wake everybody on the associated mutex in case there are | ||
1318 | - threads that have been requeued to it. | ||
1319 | - Without this, pthread_cond_destroy could block potentially | ||
1320 | - for a long time or forever, as it would depend on other | ||
1321 | - thread's using the mutex. | ||
1322 | - When all threads waiting on the mutex are woken up, pthread_cond_wait | ||
1323 | - only waits for threads to acquire and release the internal | ||
1324 | - condvar lock. */ | ||
1325 | - if (cond->__data.__mutex != NULL | ||
1326 | - && cond->__data.__mutex != (void *) ~0l) | ||
1327 | - { | ||
1328 | - pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex; | ||
1329 | - lll_futex_wake (&mut->__data.__lock, INT_MAX, | ||
1330 | - PTHREAD_MUTEX_PSHARED (mut)); | ||
1331 | - } | ||
1332 | - | ||
1333 | - do | ||
1334 | - { | ||
1335 | - lll_unlock (cond->__data.__lock, pshared); | ||
1336 | - | ||
1337 | - lll_futex_wait (&cond->__data.__nwaiters, nwaiters, pshared); | ||
1338 | - | ||
1339 | - lll_lock (cond->__data.__lock, pshared); | ||
1340 | - | ||
1341 | - nwaiters = cond->__data.__nwaiters; | ||
1342 | - } | ||
1343 | - while (nwaiters >= (1 << COND_NWAITERS_SHIFT)); | ||
1344 | + futex_wait_simple (&cond->__data.__wrefs, wrefs, private); | ||
1345 | + /* See above. */ | ||
1346 | + wrefs = atomic_load_acquire (&cond->__data.__wrefs); | ||
1347 | } | ||
1348 | - | ||
1349 | + /* The memory the condvar occupies can now be reused. */ | ||
1350 | return 0; | ||
1351 | } | ||
1352 | versioned_symbol (libpthread, __pthread_cond_destroy, | ||
1353 | diff --git a/nptl/pthread_cond_init.c b/nptl/pthread_cond_init.c | ||
1354 | index 9023370..c1eac5f 100644 | ||
1355 | --- a/nptl/pthread_cond_init.c | ||
1356 | +++ b/nptl/pthread_cond_init.c | ||
1357 | @@ -19,25 +19,29 @@ | ||
1358 | #include <shlib-compat.h> | ||
1359 | #include "pthreadP.h" | ||
1360 | #include <stap-probe.h> | ||
1361 | +#include <string.h> | ||
1362 | |||
1363 | |||
1364 | +/* See __pthread_cond_wait for details. */ | ||
1365 | int | ||
1366 | __pthread_cond_init (pthread_cond_t *cond, const pthread_condattr_t *cond_attr) | ||
1367 | { | ||
1368 | struct pthread_condattr *icond_attr = (struct pthread_condattr *) cond_attr; | ||
1369 | |||
1370 | - cond->__data.__lock = LLL_LOCK_INITIALIZER; | ||
1371 | - cond->__data.__futex = 0; | ||
1372 | - cond->__data.__nwaiters = (icond_attr != NULL | ||
1373 | - ? ((icond_attr->value >> 1) | ||
1374 | - & ((1 << COND_NWAITERS_SHIFT) - 1)) | ||
1375 | - : CLOCK_REALTIME); | ||
1376 | - cond->__data.__total_seq = 0; | ||
1377 | - cond->__data.__wakeup_seq = 0; | ||
1378 | - cond->__data.__woken_seq = 0; | ||
1379 | - cond->__data.__mutex = (icond_attr == NULL || (icond_attr->value & 1) == 0 | ||
1380 | - ? NULL : (void *) ~0l); | ||
1381 | - cond->__data.__broadcast_seq = 0; | ||
1382 | + memset (cond, 0, sizeof (pthread_cond_t)); | ||
1383 | + | ||
1384 | + /* Update the pretty printers if the internal representation of icond_attr | ||
1385 | + is changed. */ | ||
1386 | + | ||
1387 | + /* Iff not equal to ~0l, this is a PTHREAD_PROCESS_PRIVATE condvar. */ | ||
1388 | + if (icond_attr != NULL && (icond_attr->value & 1) != 0) | ||
1389 | + cond->__data.__wrefs |= __PTHREAD_COND_SHARED_MASK; | ||
1390 | + int clockid = (icond_attr != NULL | ||
1391 | + ? ((icond_attr->value >> 1) & ((1 << COND_CLOCK_BITS) - 1)) | ||
1392 | + : CLOCK_REALTIME); | ||
1393 | + /* If 0, CLOCK_REALTIME is used; CLOCK_MONOTONIC otherwise. */ | ||
1394 | + if (clockid != CLOCK_REALTIME) | ||
1395 | + cond->__data.__wrefs |= __PTHREAD_COND_CLOCK_MONOTONIC_MASK; | ||
1396 | |||
1397 | LIBC_PROBE (cond_init, 2, cond, cond_attr); | ||
1398 | |||
1399 | diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c | ||
1400 | index b3a6d3d..a95d569 100644 | ||
1401 | --- a/nptl/pthread_cond_signal.c | ||
1402 | +++ b/nptl/pthread_cond_signal.c | ||
1403 | @@ -19,62 +19,79 @@ | ||
1404 | #include <endian.h> | ||
1405 | #include <errno.h> | ||
1406 | #include <sysdep.h> | ||
1407 | -#include <lowlevellock.h> | ||
1408 | +#include <futex-internal.h> | ||
1409 | #include <pthread.h> | ||
1410 | #include <pthreadP.h> | ||
1411 | +#include <atomic.h> | ||
1412 | +#include <stdint.h> | ||
1413 | |||
1414 | #include <shlib-compat.h> | ||
1415 | -#include <kernel-features.h> | ||
1416 | #include <stap-probe.h> | ||
1417 | |||
1418 | +#include "pthread_cond_common.c" | ||
1419 | |||
1420 | +/* See __pthread_cond_wait for a high-level description of the algorithm. */ | ||
1421 | int | ||
1422 | __pthread_cond_signal (pthread_cond_t *cond) | ||
1423 | { | ||
1424 | - int pshared = (cond->__data.__mutex == (void *) ~0l) | ||
1425 | - ? LLL_SHARED : LLL_PRIVATE; | ||
1426 | - | ||
1427 | LIBC_PROBE (cond_signal, 1, cond); | ||
1428 | |||
1429 | - /* Make sure we are alone. */ | ||
1430 | - lll_lock (cond->__data.__lock, pshared); | ||
1431 | - | ||
1432 | - /* Are there any waiters to be woken? */ | ||
1433 | - if (cond->__data.__total_seq > cond->__data.__wakeup_seq) | ||
1434 | + /* First check whether there are waiters. Relaxed MO is fine for that for | ||
1435 | + the same reasons that relaxed MO is fine when observing __wseq (see | ||
1436 | + below). */ | ||
1437 | + unsigned int wrefs = atomic_load_relaxed (&cond->__data.__wrefs); | ||
1438 | + if (wrefs >> 3 == 0) | ||
1439 | + return 0; | ||
1440 | + int private = __condvar_get_private (wrefs); | ||
1441 | + | ||
1442 | + __condvar_acquire_lock (cond, private); | ||
1443 | + | ||
1444 | + /* Load the waiter sequence number, which represents our relative ordering | ||
1445 | + to any waiters. Relaxed MO is sufficient for that because: | ||
1446 | + 1) We can pick any position that is allowed by external happens-before | ||
1447 | + constraints. In particular, if another __pthread_cond_wait call | ||
1448 | + happened before us, this waiter must be eligible for being woken by | ||
1449 | + us. The only way do establish such a happens-before is by signaling | ||
1450 | + while having acquired the mutex associated with the condvar and | ||
1451 | + ensuring that the signal's critical section happens after the waiter. | ||
1452 | + Thus, the mutex ensures that we see that waiter's __wseq increase. | ||
1453 | + 2) Once we pick a position, we do not need to communicate this to the | ||
1454 | + program via a happens-before that we set up: First, any wake-up could | ||
1455 | + be a spurious wake-up, so the program must not interpret a wake-up as | ||
1456 | + an indication that the waiter happened before a particular signal; | ||
1457 | + second, a program cannot detect whether a waiter has not yet been | ||
1458 | + woken (i.e., it cannot distinguish between a non-woken waiter and one | ||
1459 | + that has been woken but hasn't resumed execution yet), and thus it | ||
1460 | + cannot try to deduce that a signal happened before a particular | ||
1461 | + waiter. */ | ||
1462 | + unsigned long long int wseq = __condvar_load_wseq_relaxed (cond); | ||
1463 | + unsigned int g1 = (wseq & 1) ^ 1; | ||
1464 | + wseq >>= 1; | ||
1465 | + bool do_futex_wake = false; | ||
1466 | + | ||
1467 | + /* If G1 is still receiving signals, we put the signal there. If not, we | ||
1468 | + check if G2 has waiters, and if so, quiesce and switch G1 to the former | ||
1469 | + G2; if this results in a new G1 with waiters (G2 might have cancellations | ||
1470 | + already, see __condvar_quiesce_and_switch_g1), we put the signal in the | ||
1471 | + new G1. */ | ||
1472 | + if ((cond->__data.__g_size[g1] != 0) | ||
1473 | + || __condvar_quiesce_and_switch_g1 (cond, wseq, &g1, private)) | ||
1474 | { | ||
1475 | - /* Yes. Mark one of them as woken. */ | ||
1476 | - ++cond->__data.__wakeup_seq; | ||
1477 | - ++cond->__data.__futex; | ||
1478 | - | ||
1479 | -#if (defined lll_futex_cmp_requeue_pi \ | ||
1480 | - && defined __ASSUME_REQUEUE_PI) | ||
1481 | - pthread_mutex_t *mut = cond->__data.__mutex; | ||
1482 | - | ||
1483 | - if (USE_REQUEUE_PI (mut) | ||
1484 | - /* This can only really fail with a ENOSYS, since nobody can modify | ||
1485 | - futex while we have the cond_lock. */ | ||
1486 | - && lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0, | ||
1487 | - &mut->__data.__lock, | ||
1488 | - cond->__data.__futex, pshared) == 0) | ||
1489 | - { | ||
1490 | - lll_unlock (cond->__data.__lock, pshared); | ||
1491 | - return 0; | ||
1492 | - } | ||
1493 | - else | ||
1494 | -#endif | ||
1495 | - /* Wake one. */ | ||
1496 | - if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, | ||
1497 | - 1, 1, | ||
1498 | - &cond->__data.__lock, | ||
1499 | - pshared), 0)) | ||
1500 | - return 0; | ||
1501 | - | ||
1502 | - /* Fallback if neither of them work. */ | ||
1503 | - lll_futex_wake (&cond->__data.__futex, 1, pshared); | ||
1504 | + /* Add a signal. Relaxed MO is fine because signaling does not need to | ||
1505 | + establish a happens-before relation (see above). We do not mask the | ||
1506 | + release-MO store when initializing a group in | ||
1507 | + __condvar_quiesce_and_switch_g1 because we use an atomic | ||
1508 | + read-modify-write and thus extend that store's release sequence. */ | ||
1509 | + atomic_fetch_add_relaxed (cond->__data.__g_signals + g1, 2); | ||
1510 | + cond->__data.__g_size[g1]--; | ||
1511 | + /* TODO Only set it if there are indeed futex waiters. */ | ||
1512 | + do_futex_wake = true; | ||
1513 | } | ||
1514 | |||
1515 | - /* We are done. */ | ||
1516 | - lll_unlock (cond->__data.__lock, pshared); | ||
1517 | + __condvar_release_lock (cond, private); | ||
1518 | + | ||
1519 | + if (do_futex_wake) | ||
1520 | + futex_wake (cond->__data.__g_signals + g1, 1, private); | ||
1521 | |||
1522 | return 0; | ||
1523 | } | ||
1524 | diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c | ||
1525 | deleted file mode 100644 | ||
1526 | index 711a51d..0000000 | ||
1527 | --- a/nptl/pthread_cond_timedwait.c | ||
1528 | +++ /dev/null | ||
1529 | @@ -1,268 +0,0 @@ | ||
1530 | -/* Copyright (C) 2003-2016 Free Software Foundation, Inc. | ||
1531 | - This file is part of the GNU C Library. | ||
1532 | - Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. | ||
1533 | - | ||
1534 | - The GNU C Library is free software; you can redistribute it and/or | ||
1535 | - modify it under the terms of the GNU Lesser General Public | ||
1536 | - License as published by the Free Software Foundation; either | ||
1537 | - version 2.1 of the License, or (at your option) any later version. | ||
1538 | - | ||
1539 | - The GNU C Library is distributed in the hope that it will be useful, | ||
1540 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
1541 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
1542 | - Lesser General Public License for more details. | ||
1543 | - | ||
1544 | - You should have received a copy of the GNU Lesser General Public | ||
1545 | - License along with the GNU C Library; if not, see | ||
1546 | - <http://www.gnu.org/licenses/>. */ | ||
1547 | - | ||
1548 | -#include <endian.h> | ||
1549 | -#include <errno.h> | ||
1550 | -#include <sysdep.h> | ||
1551 | -#include <lowlevellock.h> | ||
1552 | -#include <pthread.h> | ||
1553 | -#include <pthreadP.h> | ||
1554 | -#include <sys/time.h> | ||
1555 | -#include <kernel-features.h> | ||
1556 | - | ||
1557 | -#include <shlib-compat.h> | ||
1558 | - | ||
1559 | -#ifndef HAVE_CLOCK_GETTIME_VSYSCALL | ||
1560 | -# undef INTERNAL_VSYSCALL | ||
1561 | -# define INTERNAL_VSYSCALL INTERNAL_SYSCALL | ||
1562 | -# undef INLINE_VSYSCALL | ||
1563 | -# define INLINE_VSYSCALL INLINE_SYSCALL | ||
1564 | -#else | ||
1565 | -# include <libc-vdso.h> | ||
1566 | -#endif | ||
1567 | - | ||
1568 | -/* Cleanup handler, defined in pthread_cond_wait.c. */ | ||
1569 | -extern void __condvar_cleanup (void *arg) | ||
1570 | - __attribute__ ((visibility ("hidden"))); | ||
1571 | - | ||
1572 | -struct _condvar_cleanup_buffer | ||
1573 | -{ | ||
1574 | - int oldtype; | ||
1575 | - pthread_cond_t *cond; | ||
1576 | - pthread_mutex_t *mutex; | ||
1577 | - unsigned int bc_seq; | ||
1578 | -}; | ||
1579 | - | ||
1580 | -int | ||
1581 | -__pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, | ||
1582 | - const struct timespec *abstime) | ||
1583 | -{ | ||
1584 | - struct _pthread_cleanup_buffer buffer; | ||
1585 | - struct _condvar_cleanup_buffer cbuffer; | ||
1586 | - int result = 0; | ||
1587 | - | ||
1588 | - /* Catch invalid parameters. */ | ||
1589 | - if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) | ||
1590 | - return EINVAL; | ||
1591 | - | ||
1592 | - int pshared = (cond->__data.__mutex == (void *) ~0l) | ||
1593 | - ? LLL_SHARED : LLL_PRIVATE; | ||
1594 | - | ||
1595 | -#if (defined lll_futex_timed_wait_requeue_pi \ | ||
1596 | - && defined __ASSUME_REQUEUE_PI) | ||
1597 | - int pi_flag = 0; | ||
1598 | -#endif | ||
1599 | - | ||
1600 | - /* Make sure we are alone. */ | ||
1601 | - lll_lock (cond->__data.__lock, pshared); | ||
1602 | - | ||
1603 | - /* Now we can release the mutex. */ | ||
1604 | - int err = __pthread_mutex_unlock_usercnt (mutex, 0); | ||
1605 | - if (err) | ||
1606 | - { | ||
1607 | - lll_unlock (cond->__data.__lock, pshared); | ||
1608 | - return err; | ||
1609 | - } | ||
1610 | - | ||
1611 | - /* We have one new user of the condvar. */ | ||
1612 | - ++cond->__data.__total_seq; | ||
1613 | - ++cond->__data.__futex; | ||
1614 | - cond->__data.__nwaiters += 1 << COND_NWAITERS_SHIFT; | ||
1615 | - | ||
1616 | - /* Work around the fact that the kernel rejects negative timeout values | ||
1617 | - despite them being valid. */ | ||
1618 | - if (__glibc_unlikely (abstime->tv_sec < 0)) | ||
1619 | - goto timeout; | ||
1620 | - | ||
1621 | - /* Remember the mutex we are using here. If there is already a | ||
1622 | - different address store this is a bad user bug. Do not store | ||
1623 | - anything for pshared condvars. */ | ||
1624 | - if (cond->__data.__mutex != (void *) ~0l) | ||
1625 | - cond->__data.__mutex = mutex; | ||
1626 | - | ||
1627 | - /* Prepare structure passed to cancellation handler. */ | ||
1628 | - cbuffer.cond = cond; | ||
1629 | - cbuffer.mutex = mutex; | ||
1630 | - | ||
1631 | - /* Before we block we enable cancellation. Therefore we have to | ||
1632 | - install a cancellation handler. */ | ||
1633 | - __pthread_cleanup_push (&buffer, __condvar_cleanup, &cbuffer); | ||
1634 | - | ||
1635 | - /* The current values of the wakeup counter. The "woken" counter | ||
1636 | - must exceed this value. */ | ||
1637 | - unsigned long long int val; | ||
1638 | - unsigned long long int seq; | ||
1639 | - val = seq = cond->__data.__wakeup_seq; | ||
1640 | - /* Remember the broadcast counter. */ | ||
1641 | - cbuffer.bc_seq = cond->__data.__broadcast_seq; | ||
1642 | - | ||
1643 | - while (1) | ||
1644 | - { | ||
1645 | -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ | ||
1646 | - || !defined lll_futex_timed_wait_bitset) | ||
1647 | - struct timespec rt; | ||
1648 | - { | ||
1649 | -# ifdef __NR_clock_gettime | ||
1650 | - INTERNAL_SYSCALL_DECL (err); | ||
1651 | - (void) INTERNAL_VSYSCALL (clock_gettime, err, 2, | ||
1652 | - (cond->__data.__nwaiters | ||
1653 | - & ((1 << COND_NWAITERS_SHIFT) - 1)), | ||
1654 | - &rt); | ||
1655 | - /* Convert the absolute timeout value to a relative timeout. */ | ||
1656 | - rt.tv_sec = abstime->tv_sec - rt.tv_sec; | ||
1657 | - rt.tv_nsec = abstime->tv_nsec - rt.tv_nsec; | ||
1658 | -# else | ||
1659 | - /* Get the current time. So far we support only one clock. */ | ||
1660 | - struct timeval tv; | ||
1661 | - (void) __gettimeofday (&tv, NULL); | ||
1662 | - | ||
1663 | - /* Convert the absolute timeout value to a relative timeout. */ | ||
1664 | - rt.tv_sec = abstime->tv_sec - tv.tv_sec; | ||
1665 | - rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; | ||
1666 | -# endif | ||
1667 | - } | ||
1668 | - if (rt.tv_nsec < 0) | ||
1669 | - { | ||
1670 | - rt.tv_nsec += 1000000000; | ||
1671 | - --rt.tv_sec; | ||
1672 | - } | ||
1673 | - /* Did we already time out? */ | ||
1674 | - if (__glibc_unlikely (rt.tv_sec < 0)) | ||
1675 | - { | ||
1676 | - if (cbuffer.bc_seq != cond->__data.__broadcast_seq) | ||
1677 | - goto bc_out; | ||
1678 | - | ||
1679 | - goto timeout; | ||
1680 | - } | ||
1681 | -#endif | ||
1682 | - | ||
1683 | - unsigned int futex_val = cond->__data.__futex; | ||
1684 | - | ||
1685 | - /* Prepare to wait. Release the condvar futex. */ | ||
1686 | - lll_unlock (cond->__data.__lock, pshared); | ||
1687 | - | ||
1688 | - /* Enable asynchronous cancellation. Required by the standard. */ | ||
1689 | - cbuffer.oldtype = __pthread_enable_asynccancel (); | ||
1690 | - | ||
1691 | -/* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient | ||
1692 | - to check just the former. */ | ||
1693 | -#if (defined lll_futex_timed_wait_requeue_pi \ | ||
1694 | - && defined __ASSUME_REQUEUE_PI) | ||
1695 | - /* If pi_flag remained 1 then it means that we had the lock and the mutex | ||
1696 | - but a spurious waker raced ahead of us. Give back the mutex before | ||
1697 | - going into wait again. */ | ||
1698 | - if (pi_flag) | ||
1699 | - { | ||
1700 | - __pthread_mutex_cond_lock_adjust (mutex); | ||
1701 | - __pthread_mutex_unlock_usercnt (mutex, 0); | ||
1702 | - } | ||
1703 | - pi_flag = USE_REQUEUE_PI (mutex); | ||
1704 | - | ||
1705 | - if (pi_flag) | ||
1706 | - { | ||
1707 | - unsigned int clockbit = (cond->__data.__nwaiters & 1 | ||
1708 | - ? 0 : FUTEX_CLOCK_REALTIME); | ||
1709 | - err = lll_futex_timed_wait_requeue_pi (&cond->__data.__futex, | ||
1710 | - futex_val, abstime, clockbit, | ||
1711 | - &mutex->__data.__lock, | ||
1712 | - pshared); | ||
1713 | - pi_flag = (err == 0); | ||
1714 | - } | ||
1715 | - else | ||
1716 | -#endif | ||
1717 | - | ||
1718 | - { | ||
1719 | -#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \ | ||
1720 | - || !defined lll_futex_timed_wait_bitset) | ||
1721 | - /* Wait until woken by signal or broadcast. */ | ||
1722 | - err = lll_futex_timed_wait (&cond->__data.__futex, | ||
1723 | - futex_val, &rt, pshared); | ||
1724 | -#else | ||
1725 | - unsigned int clockbit = (cond->__data.__nwaiters & 1 | ||
1726 | - ? 0 : FUTEX_CLOCK_REALTIME); | ||
1727 | - err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val, | ||
1728 | - abstime, clockbit, pshared); | ||
1729 | -#endif | ||
1730 | - } | ||
1731 | - | ||
1732 | - /* Disable asynchronous cancellation. */ | ||
1733 | - __pthread_disable_asynccancel (cbuffer.oldtype); | ||
1734 | - | ||
1735 | - /* We are going to look at shared data again, so get the lock. */ | ||
1736 | - lll_lock (cond->__data.__lock, pshared); | ||
1737 | - | ||
1738 | - /* If a broadcast happened, we are done. */ | ||
1739 | - if (cbuffer.bc_seq != cond->__data.__broadcast_seq) | ||
1740 | - goto bc_out; | ||
1741 | - | ||
1742 | - /* Check whether we are eligible for wakeup. */ | ||
1743 | - val = cond->__data.__wakeup_seq; | ||
1744 | - if (val != seq && cond->__data.__woken_seq != val) | ||
1745 | - break; | ||
1746 | - | ||
1747 | - /* Not woken yet. Maybe the time expired? */ | ||
1748 | - if (__glibc_unlikely (err == -ETIMEDOUT)) | ||
1749 | - { | ||
1750 | - timeout: | ||
1751 | - /* Yep. Adjust the counters. */ | ||
1752 | - ++cond->__data.__wakeup_seq; | ||
1753 | - ++cond->__data.__futex; | ||
1754 | - | ||
1755 | - /* The error value. */ | ||
1756 | - result = ETIMEDOUT; | ||
1757 | - break; | ||
1758 | - } | ||
1759 | - } | ||
1760 | - | ||
1761 | - /* Another thread woken up. */ | ||
1762 | - ++cond->__data.__woken_seq; | ||
1763 | - | ||
1764 | - bc_out: | ||
1765 | - | ||
1766 | - cond->__data.__nwaiters -= 1 << COND_NWAITERS_SHIFT; | ||
1767 | - | ||
1768 | - /* If pthread_cond_destroy was called on this variable already, | ||
1769 | - notify the pthread_cond_destroy caller all waiters have left | ||
1770 | - and it can be successfully destroyed. */ | ||
1771 | - if (cond->__data.__total_seq == -1ULL | ||
1772 | - && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) | ||
1773 | - lll_futex_wake (&cond->__data.__nwaiters, 1, pshared); | ||
1774 | - | ||
1775 | - /* We are done with the condvar. */ | ||
1776 | - lll_unlock (cond->__data.__lock, pshared); | ||
1777 | - | ||
1778 | - /* The cancellation handling is back to normal, remove the handler. */ | ||
1779 | - __pthread_cleanup_pop (&buffer, 0); | ||
1780 | - | ||
1781 | - /* Get the mutex before returning. */ | ||
1782 | -#if (defined lll_futex_timed_wait_requeue_pi \ | ||
1783 | - && defined __ASSUME_REQUEUE_PI) | ||
1784 | - if (pi_flag) | ||
1785 | - { | ||
1786 | - __pthread_mutex_cond_lock_adjust (mutex); | ||
1787 | - err = 0; | ||
1788 | - } | ||
1789 | - else | ||
1790 | -#endif | ||
1791 | - err = __pthread_mutex_cond_lock (mutex); | ||
1792 | - | ||
1793 | - return err ?: result; | ||
1794 | -} | ||
1795 | - | ||
1796 | -versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, | ||
1797 | - GLIBC_2_3_2); | ||
1798 | diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c | ||
1799 | index 3f62acc..2b43402 100644 | ||
1800 | --- a/nptl/pthread_cond_wait.c | ||
1801 | +++ b/nptl/pthread_cond_wait.c | ||
1802 | @@ -19,219 +19,655 @@ | ||
1803 | #include <endian.h> | ||
1804 | #include <errno.h> | ||
1805 | #include <sysdep.h> | ||
1806 | -#include <lowlevellock.h> | ||
1807 | +#include <futex-internal.h> | ||
1808 | #include <pthread.h> | ||
1809 | #include <pthreadP.h> | ||
1810 | -#include <kernel-features.h> | ||
1811 | +#include <sys/time.h> | ||
1812 | +#include <atomic.h> | ||
1813 | +#include <stdint.h> | ||
1814 | +#include <stdbool.h> | ||
1815 | |||
1816 | #include <shlib-compat.h> | ||
1817 | #include <stap-probe.h> | ||
1818 | +#include <time.h> | ||
1819 | + | ||
1820 | +#include "pthread_cond_common.c" | ||
1821 | + | ||
1822 | |||
1823 | struct _condvar_cleanup_buffer | ||
1824 | { | ||
1825 | - int oldtype; | ||
1826 | + uint64_t wseq; | ||
1827 | pthread_cond_t *cond; | ||
1828 | pthread_mutex_t *mutex; | ||
1829 | - unsigned int bc_seq; | ||
1830 | + int private; | ||
1831 | }; | ||
1832 | |||
1833 | |||
1834 | -void | ||
1835 | -__attribute__ ((visibility ("hidden"))) | ||
1836 | -__condvar_cleanup (void *arg) | ||
1837 | +/* Decrease the waiter reference count. */ | ||
1838 | +static void | ||
1839 | +__condvar_confirm_wakeup (pthread_cond_t *cond, int private) | ||
1840 | { | ||
1841 | - struct _condvar_cleanup_buffer *cbuffer = | ||
1842 | - (struct _condvar_cleanup_buffer *) arg; | ||
1843 | - unsigned int destroying; | ||
1844 | - int pshared = (cbuffer->cond->__data.__mutex == (void *) ~0l) | ||
1845 | - ? LLL_SHARED : LLL_PRIVATE; | ||
1846 | + /* If destruction is pending (i.e., the wake-request flag is nonzero) and we | ||
1847 | + are the last waiter (prior value of __wrefs was 1 << 3), then wake any | ||
1848 | + threads waiting in pthread_cond_destroy. Release MO to synchronize with | ||
1849 | + these threads. Don't bother clearing the wake-up request flag. */ | ||
1850 | + if ((atomic_fetch_add_release (&cond->__data.__wrefs, -8) >> 2) == 3) | ||
1851 | + futex_wake (&cond->__data.__wrefs, INT_MAX, private); | ||
1852 | +} | ||
1853 | + | ||
1854 | |||
1855 | - /* We are going to modify shared data. */ | ||
1856 | - lll_lock (cbuffer->cond->__data.__lock, pshared); | ||
1857 | +/* Cancel waiting after having registered as a waiter previously. SEQ is our | ||
1858 | + position and G is our group index. | ||
1859 | + The goal of cancellation is to make our group smaller if that is still | ||
1860 | + possible. If we are in a closed group, this is not possible anymore; in | ||
1861 | + this case, we need to send a replacement signal for the one we effectively | ||
1862 | + consumed because the signal should have gotten consumed by another waiter | ||
1863 | + instead; we must not both cancel waiting and consume a signal. | ||
1864 | + | ||
1865 | + Must not be called while still holding a reference on the group. | ||
1866 | + | ||
1867 | + Returns true iff we consumed a signal. | ||
1868 | + | ||
1869 | + On some kind of timeouts, we may be able to pretend that a signal we | ||
1870 | + effectively consumed happened before the timeout (i.e., similarly to first | ||
1871 | + spinning on signals before actually checking whether the timeout has | ||
1872 | + passed already). Doing this would allow us to skip sending a replacement | ||
1873 | + signal, but this case might happen rarely because the end of the timeout | ||
1874 | + must race with someone else sending a signal. Therefore, we don't bother | ||
1875 | + trying to optimize this. */ | ||
1876 | +static void | ||
1877 | +__condvar_cancel_waiting (pthread_cond_t *cond, uint64_t seq, unsigned int g, | ||
1878 | + int private) | ||
1879 | +{ | ||
1880 | + bool consumed_signal = false; | ||
1881 | |||
1882 | - if (cbuffer->bc_seq == cbuffer->cond->__data.__broadcast_seq) | ||
1883 | + /* No deadlock with group switching is possible here because we have do | ||
1884 | + not hold a reference on the group. */ | ||
1885 | + __condvar_acquire_lock (cond, private); | ||
1886 | + | ||
1887 | + uint64_t g1_start = __condvar_load_g1_start_relaxed (cond) >> 1; | ||
1888 | + if (g1_start > seq) | ||
1889 | + { | ||
1890 | + /* Our group is closed, so someone provided enough signals for it. | ||
1891 | + Thus, we effectively consumed a signal. */ | ||
1892 | + consumed_signal = true; | ||
1893 | + } | ||
1894 | + else | ||
1895 | { | ||
1896 | - /* This thread is not waiting anymore. Adjust the sequence counters | ||
1897 | - appropriately. We do not increment WAKEUP_SEQ if this would | ||
1898 | - bump it over the value of TOTAL_SEQ. This can happen if a thread | ||
1899 | - was woken and then canceled. */ | ||
1900 | - if (cbuffer->cond->__data.__wakeup_seq | ||
1901 | - < cbuffer->cond->__data.__total_seq) | ||
1902 | + if (g1_start + __condvar_get_orig_size (cond) <= seq) | ||
1903 | + { | ||
1904 | + /* We are in the current G2 and thus cannot have consumed a signal. | ||
1905 | + Reduce its effective size or handle overflow. Remember that in | ||
1906 | + G2, unsigned int size is zero or a negative value. */ | ||
1907 | + if (cond->__data.__g_size[g] + __PTHREAD_COND_MAX_GROUP_SIZE > 0) | ||
1908 | + { | ||
1909 | + cond->__data.__g_size[g]--; | ||
1910 | + } | ||
1911 | + else | ||
1912 | + { | ||
1913 | + /* Cancellations would overflow the maximum group size. Just | ||
1914 | + wake up everyone spuriously to create a clean state. This | ||
1915 | + also means we do not consume a signal someone else sent. */ | ||
1916 | + __condvar_release_lock (cond, private); | ||
1917 | + __pthread_cond_broadcast (cond); | ||
1918 | + return; | ||
1919 | + } | ||
1920 | + } | ||
1921 | + else | ||
1922 | { | ||
1923 | - ++cbuffer->cond->__data.__wakeup_seq; | ||
1924 | - ++cbuffer->cond->__data.__futex; | ||
1925 | + /* We are in current G1. If the group's size is zero, someone put | ||
1926 | + a signal in the group that nobody else but us can consume. */ | ||
1927 | + if (cond->__data.__g_size[g] == 0) | ||
1928 | + consumed_signal = true; | ||
1929 | + else | ||
1930 | + { | ||
1931 | + /* Otherwise, we decrease the size of the group. This is | ||
1932 | + equivalent to atomically putting in a signal just for us and | ||
1933 | + consuming it right away. We do not consume a signal sent | ||
1934 | + by someone else. We also cannot have consumed a futex | ||
1935 | + wake-up because if we were cancelled or timed out in a futex | ||
1936 | + call, the futex will wake another waiter. */ | ||
1937 | + cond->__data.__g_size[g]--; | ||
1938 | + } | ||
1939 | } | ||
1940 | - ++cbuffer->cond->__data.__woken_seq; | ||
1941 | } | ||
1942 | |||
1943 | - cbuffer->cond->__data.__nwaiters -= 1 << COND_NWAITERS_SHIFT; | ||
1944 | + __condvar_release_lock (cond, private); | ||
1945 | |||
1946 | - /* If pthread_cond_destroy was called on this variable already, | ||
1947 | - notify the pthread_cond_destroy caller all waiters have left | ||
1948 | - and it can be successfully destroyed. */ | ||
1949 | - destroying = 0; | ||
1950 | - if (cbuffer->cond->__data.__total_seq == -1ULL | ||
1951 | - && cbuffer->cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) | ||
1952 | + if (consumed_signal) | ||
1953 | { | ||
1954 | - lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1, pshared); | ||
1955 | - destroying = 1; | ||
1956 | + /* We effectively consumed a signal even though we didn't want to. | ||
1957 | + Therefore, we need to send a replacement signal. | ||
1958 | + If we would want to optimize this, we could do what | ||
1959 | + pthread_cond_signal does right in the critical section above. */ | ||
1960 | + __pthread_cond_signal (cond); | ||
1961 | } | ||
1962 | +} | ||
1963 | |||
1964 | - /* We are done. */ | ||
1965 | - lll_unlock (cbuffer->cond->__data.__lock, pshared); | ||
1966 | - | ||
1967 | - /* Wake everybody to make sure no condvar signal gets lost. */ | ||
1968 | - if (! destroying) | ||
1969 | - lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared); | ||
1970 | - | ||
1971 | - /* Get the mutex before returning unless asynchronous cancellation | ||
1972 | - is in effect. We don't try to get the mutex if we already own it. */ | ||
1973 | - if (!(USE_REQUEUE_PI (cbuffer->mutex)) | ||
1974 | - || ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK) | ||
1975 | - != THREAD_GETMEM (THREAD_SELF, tid))) | ||
1976 | - { | ||
1977 | - __pthread_mutex_cond_lock (cbuffer->mutex); | ||
1978 | - } | ||
1979 | - else | ||
1980 | - __pthread_mutex_cond_lock_adjust (cbuffer->mutex); | ||
1981 | +/* Wake up any signalers that might be waiting. */ | ||
1982 | +static void | ||
1983 | +__condvar_dec_grefs (pthread_cond_t *cond, unsigned int g, int private) | ||
1984 | +{ | ||
1985 | + /* Release MO to synchronize-with the acquire load in | ||
1986 | + __condvar_quiesce_and_switch_g1. */ | ||
1987 | + if (atomic_fetch_add_release (cond->__data.__g_refs + g, -2) == 3) | ||
1988 | + { | ||
1989 | + /* Clear the wake-up request flag before waking up. We do not need more | ||
1990 | + than relaxed MO and it doesn't matter if we apply this for an aliased | ||
1991 | + group because we wake all futex waiters right after clearing the | ||
1992 | + flag. */ | ||
1993 | + atomic_fetch_and_relaxed (cond->__data.__g_refs + g, ~(unsigned int) 1); | ||
1994 | + futex_wake (cond->__data.__g_refs + g, INT_MAX, private); | ||
1995 | + } | ||
1996 | } | ||
1997 | |||
1998 | +/* Clean-up for cancellation of waiters waiting for normal signals. We cancel | ||
1999 | + our registration as a waiter, confirm we have woken up, and re-acquire the | ||
2000 | + mutex. */ | ||
2001 | +static void | ||
2002 | +__condvar_cleanup_waiting (void *arg) | ||
2003 | +{ | ||
2004 | + struct _condvar_cleanup_buffer *cbuffer = | ||
2005 | + (struct _condvar_cleanup_buffer *) arg; | ||
2006 | + pthread_cond_t *cond = cbuffer->cond; | ||
2007 | + unsigned g = cbuffer->wseq & 1; | ||
2008 | |||
2009 | -int | ||
2010 | -__pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) | ||
2011 | + __condvar_dec_grefs (cond, g, cbuffer->private); | ||
2012 | + | ||
2013 | + __condvar_cancel_waiting (cond, cbuffer->wseq >> 1, g, cbuffer->private); | ||
2014 | + /* FIXME With the current cancellation implementation, it is possible that | ||
2015 | + a thread is cancelled after it has returned from a syscall. This could | ||
2016 | + result in a cancelled waiter consuming a futex wake-up that is then | ||
2017 | + causing another waiter in the same group to not wake up. To work around | ||
2018 | + this issue until we have fixed cancellation, just add a futex wake-up | ||
2019 | + conservatively. */ | ||
2020 | + futex_wake (cond->__data.__g_signals + g, 1, cbuffer->private); | ||
2021 | + | ||
2022 | + __condvar_confirm_wakeup (cond, cbuffer->private); | ||
2023 | + | ||
2024 | + /* XXX If locking the mutex fails, should we just stop execution? This | ||
2025 | + might be better than silently ignoring the error. */ | ||
2026 | + __pthread_mutex_cond_lock (cbuffer->mutex); | ||
2027 | +} | ||
2028 | + | ||
2029 | +/* This condvar implementation guarantees that all calls to signal and | ||
2030 | + broadcast and all of the three virtually atomic parts of each call to wait | ||
2031 | + (i.e., (1) releasing the mutex and blocking, (2) unblocking, and (3) re- | ||
2032 | + acquiring the mutex) happen in some total order that is consistent with the | ||
2033 | + happens-before relations in the calling program. However, this order does | ||
2034 | + not necessarily result in additional happens-before relations being | ||
2035 | + established (which aligns well with spurious wake-ups being allowed). | ||
2036 | + | ||
2037 | + All waiters acquire a certain position in a 64b waiter sequence (__wseq). | ||
2038 | + This sequence determines which waiters are allowed to consume signals. | ||
2039 | + A broadcast is equal to sending as many signals as are unblocked waiters. | ||
2040 | + When a signal arrives, it samples the current value of __wseq with a | ||
2041 | + relaxed-MO load (i.e., the position the next waiter would get). (This is | ||
2042 | + sufficient because it is consistent with happens-before; the caller can | ||
2043 | + enforce stronger ordering constraints by calling signal while holding the | ||
2044 | + mutex.) Only waiters with a position less than the __wseq value observed | ||
2045 | + by the signal are eligible to consume this signal. | ||
2046 | + | ||
2047 | + This would be straight-forward to implement if waiters would just spin but | ||
2048 | + we need to let them block using futexes. Futexes give no guarantee of | ||
2049 | + waking in FIFO order, so we cannot reliably wake eligible waiters if we | ||
2050 | + just use a single futex. Also, futex words are 32b in size, but we need | ||
2051 | + to distinguish more than 1<<32 states because we need to represent the | ||
2052 | + order of wake-up (and thus which waiters are eligible to consume signals); | ||
2053 | + blocking in a futex is not atomic with a waiter determining its position in | ||
2054 | + the waiter sequence, so we need the futex word to reliably notify waiters | ||
2055 | + that they should not attempt to block anymore because they have been | ||
2056 | + already signaled in the meantime. While an ABA issue on a 32b value will | ||
2057 | + be rare, ignoring it when we are aware of it is not the right thing to do | ||
2058 | + either. | ||
2059 | + | ||
2060 | + Therefore, we use a 64b counter to represent the waiter sequence (on | ||
2061 | + architectures which only support 32b atomics, we use a few bits less). | ||
2062 | + To deal with the blocking using futexes, we maintain two groups of waiters: | ||
2063 | + * Group G1 consists of waiters that are all eligible to consume signals; | ||
2064 | + incoming signals will always signal waiters in this group until all | ||
2065 | + waiters in G1 have been signaled. | ||
2066 | + * Group G2 consists of waiters that arrive when a G1 is present and still | ||
2067 | + contains waiters that have not been signaled. When all waiters in G1 | ||
2068 | + are signaled and a new signal arrives, the new signal will convert G2 | ||
2069 | + into the new G1 and create a new G2 for future waiters. | ||
2070 | + | ||
2071 | + We cannot allocate new memory because of process-shared condvars, so we | ||
2072 | + have just two slots of groups that change their role between G1 and G2. | ||
2073 | + Each has a separate futex word, a number of signals available for | ||
2074 | + consumption, a size (number of waiters in the group that have not been | ||
2075 | + signaled), and a reference count. | ||
2076 | + | ||
2077 | + The group reference count is used to maintain the number of waiters that | ||
2078 | + are using the group's futex. Before a group can change its role, the | ||
2079 | + reference count must show that no waiters are using the futex anymore; this | ||
2080 | + prevents ABA issues on the futex word. | ||
2081 | + | ||
2082 | + To represent which intervals in the waiter sequence the groups cover (and | ||
2083 | + thus also which group slot contains G1 or G2), we use a 64b counter to | ||
2084 | + designate the start position of G1 (inclusive), and a single bit in the | ||
2085 | + waiter sequence counter to represent which group slot currently contains | ||
2086 | + G2. This allows us to switch group roles atomically wrt. waiters obtaining | ||
2087 | + a position in the waiter sequence. The G1 start position allows waiters to | ||
2088 | + figure out whether they are in a group that has already been completely | ||
2089 | + signaled (i.e., if the current G1 starts at a later position that the | ||
2090 | + waiter's position). Waiters cannot determine whether they are currently | ||
2091 | + in G2 or G1 -- but they do not have too because all they are interested in | ||
2092 | + is whether there are available signals, and they always start in G2 (whose | ||
2093 | + group slot they know because of the bit in the waiter sequence. Signalers | ||
2094 | + will simply fill the right group until it is completely signaled and can | ||
2095 | + be closed (they do not switch group roles until they really have to to | ||
2096 | + decrease the likelihood of having to wait for waiters still holding a | ||
2097 | + reference on the now-closed G1). | ||
2098 | + | ||
2099 | + Signalers maintain the initial size of G1 to be able to determine where | ||
2100 | + G2 starts (G2 is always open-ended until it becomes G1). They track the | ||
2101 | + remaining size of a group; when waiters cancel waiting (due to PThreads | ||
2102 | + cancellation or timeouts), they will decrease this remaining size as well. | ||
2103 | + | ||
2104 | + To implement condvar destruction requirements (i.e., that | ||
2105 | + pthread_cond_destroy can be called as soon as all waiters have been | ||
2106 | + signaled), waiters increment a reference count before starting to wait and | ||
2107 | + decrement it after they stopped waiting but right before they acquire the | ||
2108 | + mutex associated with the condvar. | ||
2109 | + | ||
2110 | + pthread_cond_t thus consists of the following (bits that are used for | ||
2111 | + flags and are not part of the primary value of each field but necessary | ||
2112 | + to make some things atomic or because there was no space for them | ||
2113 | + elsewhere in the data structure): | ||
2114 | + | ||
2115 | + __wseq: Waiter sequence counter | ||
2116 | + * LSB is index of current G2. | ||
2117 | + * Waiters fetch-add while having acquire the mutex associated with the | ||
2118 | + condvar. Signalers load it and fetch-xor it concurrently. | ||
2119 | + __g1_start: Starting position of G1 (inclusive) | ||
2120 | + * LSB is index of current G2. | ||
2121 | + * Modified by signalers while having acquired the condvar-internal lock | ||
2122 | + and observed concurrently by waiters. | ||
2123 | + __g1_orig_size: Initial size of G1 | ||
2124 | + * The two least-significant bits represent the condvar-internal lock. | ||
2125 | + * Only accessed while having acquired the condvar-internal lock. | ||
2126 | + __wrefs: Waiter reference counter. | ||
2127 | + * Bit 2 is true if waiters should run futex_wake when they remove the | ||
2128 | + last reference. pthread_cond_destroy uses this as futex word. | ||
2129 | + * Bit 1 is the clock ID (0 == CLOCK_REALTIME, 1 == CLOCK_MONOTONIC). | ||
2130 | + * Bit 0 is true iff this is a process-shared condvar. | ||
2131 | + * Simple reference count used by both waiters and pthread_cond_destroy. | ||
2132 | + (If the format of __wrefs is changed, update nptl_lock_constants.pysym | ||
2133 | + and the pretty printers.) | ||
2134 | + For each of the two groups, we have: | ||
2135 | + __g_refs: Futex waiter reference count. | ||
2136 | + * LSB is true if waiters should run futex_wake when they remove the | ||
2137 | + last reference. | ||
2138 | + * Reference count used by waiters concurrently with signalers that have | ||
2139 | + acquired the condvar-internal lock. | ||
2140 | + __g_signals: The number of signals that can still be consumed. | ||
2141 | + * Used as a futex word by waiters. Used concurrently by waiters and | ||
2142 | + signalers. | ||
2143 | + * LSB is true iff this group has been completely signaled (i.e., it is | ||
2144 | + closed). | ||
2145 | + __g_size: Waiters remaining in this group (i.e., which have not been | ||
2146 | + signaled yet. | ||
2147 | + * Accessed by signalers and waiters that cancel waiting (both do so only | ||
2148 | + when having acquired the condvar-internal lock. | ||
2149 | + * The size of G2 is always zero because it cannot be determined until | ||
2150 | + the group becomes G1. | ||
2151 | + * Although this is of unsigned type, we rely on using unsigned overflow | ||
2152 | + rules to make this hold effectively negative values too (in | ||
2153 | + particular, when waiters in G2 cancel waiting). | ||
2154 | + | ||
2155 | + A PTHREAD_COND_INITIALIZER condvar has all fields set to zero, which yields | ||
2156 | + a condvar that has G2 starting at position 0 and a G1 that is closed. | ||
2157 | + | ||
2158 | + Because waiters do not claim ownership of a group right when obtaining a | ||
2159 | + position in __wseq but only reference count the group when using futexes | ||
2160 | + to block, it can happen that a group gets closed before a waiter can | ||
2161 | + increment the reference count. Therefore, waiters have to check whether | ||
2162 | + their group is already closed using __g1_start. They also have to perform | ||
2163 | + this check when spinning when trying to grab a signal from __g_signals. | ||
2164 | + Note that for these checks, using relaxed MO to load __g1_start is | ||
2165 | + sufficient because if a waiter can see a sufficiently large value, it could | ||
2166 | + have also consume a signal in the waiters group. | ||
2167 | + | ||
2168 | + Waiters try to grab a signal from __g_signals without holding a reference | ||
2169 | + count, which can lead to stealing a signal from a more recent group after | ||
2170 | + their own group was already closed. They cannot always detect whether they | ||
2171 | + in fact did because they do not know when they stole, but they can | ||
2172 | + conservatively add a signal back to the group they stole from; if they | ||
2173 | + did so unnecessarily, all that happens is a spurious wake-up. To make this | ||
2174 | + even less likely, __g1_start contains the index of the current g2 too, | ||
2175 | + which allows waiters to check if there aliasing on the group slots; if | ||
2176 | + there wasn't, they didn't steal from the current G1, which means that the | ||
2177 | + G1 they stole from must have been already closed and they do not need to | ||
2178 | + fix anything. | ||
2179 | + | ||
2180 | + It is essential that the last field in pthread_cond_t is __g_signals[1]: | ||
2181 | + The previous condvar used a pointer-sized field in pthread_cond_t, so a | ||
2182 | + PTHREAD_COND_INITIALIZER from that condvar implementation might only | ||
2183 | + initialize 4 bytes to zero instead of the 8 bytes we need (i.e., 44 bytes | ||
2184 | + in total instead of the 48 we need). __g_signals[1] is not accessed before | ||
2185 | + the first group switch (G2 starts at index 0), which will set its value to | ||
2186 | + zero after a harmless fetch-or whose return value is ignored. This | ||
2187 | + effectively completes initialization. | ||
2188 | + | ||
2189 | + | ||
2190 | + Limitations: | ||
2191 | + * This condvar isn't designed to allow for more than | ||
2192 | + __PTHREAD_COND_MAX_GROUP_SIZE * (1 << 31) calls to __pthread_cond_wait. | ||
2193 | + * More than __PTHREAD_COND_MAX_GROUP_SIZE concurrent waiters are not | ||
2194 | + supported. | ||
2195 | + * Beyond what is allowed as errors by POSIX or documented, we can also | ||
2196 | + return the following errors: | ||
2197 | + * EPERM if MUTEX is a recursive mutex and the caller doesn't own it. | ||
2198 | + * EOWNERDEAD or ENOTRECOVERABLE when using robust mutexes. Unlike | ||
2199 | + for other errors, this can happen when we re-acquire the mutex; this | ||
2200 | + isn't allowed by POSIX (which requires all errors to virtually happen | ||
2201 | + before we release the mutex or change the condvar state), but there's | ||
2202 | + nothing we can do really. | ||
2203 | + * When using PTHREAD_MUTEX_PP_* mutexes, we can also return all errors | ||
2204 | + returned by __pthread_tpp_change_priority. We will already have | ||
2205 | + released the mutex in such cases, so the caller cannot expect to own | ||
2206 | + MUTEX. | ||
2207 | + | ||
2208 | + Other notes: | ||
2209 | + * Instead of the normal mutex unlock / lock functions, we use | ||
2210 | + __pthread_mutex_unlock_usercnt(m, 0) / __pthread_mutex_cond_lock(m) | ||
2211 | + because those will not change the mutex-internal users count, so that it | ||
2212 | + can be detected when a condvar is still associated with a particular | ||
2213 | + mutex because there is a waiter blocked on this condvar using this mutex. | ||
2214 | +*/ | ||
2215 | +static __always_inline int | ||
2216 | +__pthread_cond_wait_common (pthread_cond_t *cond, pthread_mutex_t *mutex, | ||
2217 | + const struct timespec *abstime) | ||
2218 | { | ||
2219 | - struct _pthread_cleanup_buffer buffer; | ||
2220 | - struct _condvar_cleanup_buffer cbuffer; | ||
2221 | + const int maxspin = 0; | ||
2222 | int err; | ||
2223 | - int pshared = (cond->__data.__mutex == (void *) ~0l) | ||
2224 | - ? LLL_SHARED : LLL_PRIVATE; | ||
2225 | - | ||
2226 | -#if (defined lll_futex_wait_requeue_pi \ | ||
2227 | - && defined __ASSUME_REQUEUE_PI) | ||
2228 | - int pi_flag = 0; | ||
2229 | -#endif | ||
2230 | + int result = 0; | ||
2231 | |||
2232 | LIBC_PROBE (cond_wait, 2, cond, mutex); | ||
2233 | |||
2234 | - /* Make sure we are alone. */ | ||
2235 | - lll_lock (cond->__data.__lock, pshared); | ||
2236 | - | ||
2237 | - /* Now we can release the mutex. */ | ||
2238 | + /* Acquire a position (SEQ) in the waiter sequence (WSEQ). We use an | ||
2239 | + atomic operation because signals and broadcasts may update the group | ||
2240 | + switch without acquiring the mutex. We do not need release MO here | ||
2241 | + because we do not need to establish any happens-before relation with | ||
2242 | + signalers (see __pthread_cond_signal); modification order alone | ||
2243 | + establishes a total order of waiters/signals. We do need acquire MO | ||
2244 | + to synchronize with group reinitialization in | ||
2245 | + __condvar_quiesce_and_switch_g1. */ | ||
2246 | + uint64_t wseq = __condvar_fetch_add_wseq_acquire (cond, 2); | ||
2247 | + /* Find our group's index. We always go into what was G2 when we acquired | ||
2248 | + our position. */ | ||
2249 | + unsigned int g = wseq & 1; | ||
2250 | + uint64_t seq = wseq >> 1; | ||
2251 | + | ||
2252 | + /* Increase the waiter reference count. Relaxed MO is sufficient because | ||
2253 | + we only need to synchronize when decrementing the reference count. */ | ||
2254 | + unsigned int flags = atomic_fetch_add_relaxed (&cond->__data.__wrefs, 8); | ||
2255 | + int private = __condvar_get_private (flags); | ||
2256 | + | ||
2257 | + /* Now that we are registered as a waiter, we can release the mutex. | ||
2258 | + Waiting on the condvar must be atomic with releasing the mutex, so if | ||
2259 | + the mutex is used to establish a happens-before relation with any | ||
2260 | + signaler, the waiter must be visible to the latter; thus, we release the | ||
2261 | + mutex after registering as waiter. | ||
2262 | + If releasing the mutex fails, we just cancel our registration as a | ||
2263 | + waiter and confirm that we have woken up. */ | ||
2264 | err = __pthread_mutex_unlock_usercnt (mutex, 0); | ||
2265 | - if (__glibc_unlikely (err)) | ||
2266 | + if (__glibc_unlikely (err != 0)) | ||
2267 | { | ||
2268 | - lll_unlock (cond->__data.__lock, pshared); | ||
2269 | + __condvar_cancel_waiting (cond, seq, g, private); | ||
2270 | + __condvar_confirm_wakeup (cond, private); | ||
2271 | return err; | ||
2272 | } | ||
2273 | |||
2274 | - /* We have one new user of the condvar. */ | ||
2275 | - ++cond->__data.__total_seq; | ||
2276 | - ++cond->__data.__futex; | ||
2277 | - cond->__data.__nwaiters += 1 << COND_NWAITERS_SHIFT; | ||
2278 | - | ||
2279 | - /* Remember the mutex we are using here. If there is already a | ||
2280 | - different address store this is a bad user bug. Do not store | ||
2281 | - anything for pshared condvars. */ | ||
2282 | - if (cond->__data.__mutex != (void *) ~0l) | ||
2283 | - cond->__data.__mutex = mutex; | ||
2284 | - | ||
2285 | - /* Prepare structure passed to cancellation handler. */ | ||
2286 | - cbuffer.cond = cond; | ||
2287 | - cbuffer.mutex = mutex; | ||
2288 | - | ||
2289 | - /* Before we block we enable cancellation. Therefore we have to | ||
2290 | - install a cancellation handler. */ | ||
2291 | - __pthread_cleanup_push (&buffer, __condvar_cleanup, &cbuffer); | ||
2292 | - | ||
2293 | - /* The current values of the wakeup counter. The "woken" counter | ||
2294 | - must exceed this value. */ | ||
2295 | - unsigned long long int val; | ||
2296 | - unsigned long long int seq; | ||
2297 | - val = seq = cond->__data.__wakeup_seq; | ||
2298 | - /* Remember the broadcast counter. */ | ||
2299 | - cbuffer.bc_seq = cond->__data.__broadcast_seq; | ||
2300 | + /* Now wait until a signal is available in our group or it is closed. | ||
2301 | + Acquire MO so that if we observe a value of zero written after group | ||
2302 | + switching in __condvar_quiesce_and_switch_g1, we synchronize with that | ||
2303 | + store and will see the prior update of __g1_start done while switching | ||
2304 | + groups too. */ | ||
2305 | + unsigned int signals = atomic_load_acquire (cond->__data.__g_signals + g); | ||
2306 | |||
2307 | do | ||
2308 | { | ||
2309 | - unsigned int futex_val = cond->__data.__futex; | ||
2310 | - /* Prepare to wait. Release the condvar futex. */ | ||
2311 | - lll_unlock (cond->__data.__lock, pshared); | ||
2312 | - | ||
2313 | - /* Enable asynchronous cancellation. Required by the standard. */ | ||
2314 | - cbuffer.oldtype = __pthread_enable_asynccancel (); | ||
2315 | - | ||
2316 | -#if (defined lll_futex_wait_requeue_pi \ | ||
2317 | - && defined __ASSUME_REQUEUE_PI) | ||
2318 | - /* If pi_flag remained 1 then it means that we had the lock and the mutex | ||
2319 | - but a spurious waker raced ahead of us. Give back the mutex before | ||
2320 | - going into wait again. */ | ||
2321 | - if (pi_flag) | ||
2322 | + while (1) | ||
2323 | { | ||
2324 | - __pthread_mutex_cond_lock_adjust (mutex); | ||
2325 | - __pthread_mutex_unlock_usercnt (mutex, 0); | ||
2326 | + /* Spin-wait first. | ||
2327 | + Note that spinning first without checking whether a timeout | ||
2328 | + passed might lead to what looks like a spurious wake-up even | ||
2329 | + though we should return ETIMEDOUT (e.g., if the caller provides | ||
2330 | + an absolute timeout that is clearly in the past). However, | ||
2331 | + (1) spurious wake-ups are allowed, (2) it seems unlikely that a | ||
2332 | + user will (ab)use pthread_cond_wait as a check for whether a | ||
2333 | + point in time is in the past, and (3) spinning first without | ||
2334 | + having to compare against the current time seems to be the right | ||
2335 | + choice from a performance perspective for most use cases. */ | ||
2336 | + unsigned int spin = maxspin; | ||
2337 | + while (signals == 0 && spin > 0) | ||
2338 | + { | ||
2339 | + /* Check that we are not spinning on a group that's already | ||
2340 | + closed. */ | ||
2341 | + if (seq < (__condvar_load_g1_start_relaxed (cond) >> 1)) | ||
2342 | + goto done; | ||
2343 | + | ||
2344 | + /* TODO Back off. */ | ||
2345 | + | ||
2346 | + /* Reload signals. See above for MO. */ | ||
2347 | + signals = atomic_load_acquire (cond->__data.__g_signals + g); | ||
2348 | + spin--; | ||
2349 | + } | ||
2350 | + | ||
2351 | + /* If our group will be closed as indicated by the flag on signals, | ||
2352 | + don't bother grabbing a signal. */ | ||
2353 | + if (signals & 1) | ||
2354 | + goto done; | ||
2355 | + | ||
2356 | + /* If there is an available signal, don't block. */ | ||
2357 | + if (signals != 0) | ||
2358 | + break; | ||
2359 | + | ||
2360 | + /* No signals available after spinning, so prepare to block. | ||
2361 | + We first acquire a group reference and use acquire MO for that so | ||
2362 | + that we synchronize with the dummy read-modify-write in | ||
2363 | + __condvar_quiesce_and_switch_g1 if we read from that. In turn, | ||
2364 | + in this case this will make us see the closed flag on __g_signals | ||
2365 | + that designates a concurrent attempt to reuse the group's slot. | ||
2366 | + We use acquire MO for the __g_signals check to make the | ||
2367 | + __g1_start check work (see spinning above). | ||
2368 | + Note that the group reference acquisition will not mask the | ||
2369 | + release MO when decrementing the reference count because we use | ||
2370 | + an atomic read-modify-write operation and thus extend the release | ||
2371 | + sequence. */ | ||
2372 | + atomic_fetch_add_acquire (cond->__data.__g_refs + g, 2); | ||
2373 | + if (((atomic_load_acquire (cond->__data.__g_signals + g) & 1) != 0) | ||
2374 | + || (seq < (__condvar_load_g1_start_relaxed (cond) >> 1))) | ||
2375 | + { | ||
2376 | + /* Our group is closed. Wake up any signalers that might be | ||
2377 | + waiting. */ | ||
2378 | + __condvar_dec_grefs (cond, g, private); | ||
2379 | + goto done; | ||
2380 | + } | ||
2381 | + | ||
2382 | + // Now block. | ||
2383 | + struct _pthread_cleanup_buffer buffer; | ||
2384 | + struct _condvar_cleanup_buffer cbuffer; | ||
2385 | + cbuffer.wseq = wseq; | ||
2386 | + cbuffer.cond = cond; | ||
2387 | + cbuffer.mutex = mutex; | ||
2388 | + cbuffer.private = private; | ||
2389 | + __pthread_cleanup_push (&buffer, __condvar_cleanup_waiting, &cbuffer); | ||
2390 | + | ||
2391 | + if (abstime == NULL) | ||
2392 | + { | ||
2393 | + /* Block without a timeout. */ | ||
2394 | + err = futex_wait_cancelable ( | ||
2395 | + cond->__data.__g_signals + g, 0, private); | ||
2396 | + } | ||
2397 | + else | ||
2398 | + { | ||
2399 | + /* Block, but with a timeout. | ||
2400 | + Work around the fact that the kernel rejects negative timeout | ||
2401 | + values despite them being valid. */ | ||
2402 | + if (__glibc_unlikely (abstime->tv_sec < 0)) | ||
2403 | + err = ETIMEDOUT; | ||
2404 | + | ||
2405 | + else if ((flags & __PTHREAD_COND_CLOCK_MONOTONIC_MASK) != 0) | ||
2406 | + { | ||
2407 | + /* CLOCK_MONOTONIC is requested. */ | ||
2408 | + struct timespec rt; | ||
2409 | + if (__clock_gettime (CLOCK_MONOTONIC, &rt) != 0) | ||
2410 | + __libc_fatal ("clock_gettime does not support " | ||
2411 | + "CLOCK_MONOTONIC"); | ||
2412 | + /* Convert the absolute timeout value to a relative | ||
2413 | + timeout. */ | ||
2414 | + rt.tv_sec = abstime->tv_sec - rt.tv_sec; | ||
2415 | + rt.tv_nsec = abstime->tv_nsec - rt.tv_nsec; | ||
2416 | + if (rt.tv_nsec < 0) | ||
2417 | + { | ||
2418 | + rt.tv_nsec += 1000000000; | ||
2419 | + --rt.tv_sec; | ||
2420 | + } | ||
2421 | + /* Did we already time out? */ | ||
2422 | + if (__glibc_unlikely (rt.tv_sec < 0)) | ||
2423 | + err = ETIMEDOUT; | ||
2424 | + else | ||
2425 | + err = futex_reltimed_wait_cancelable | ||
2426 | + (cond->__data.__g_signals + g, 0, &rt, private); | ||
2427 | + } | ||
2428 | + else | ||
2429 | + { | ||
2430 | + /* Use CLOCK_REALTIME. */ | ||
2431 | + err = futex_abstimed_wait_cancelable | ||
2432 | + (cond->__data.__g_signals + g, 0, abstime, private); | ||
2433 | + } | ||
2434 | + } | ||
2435 | + | ||
2436 | + __pthread_cleanup_pop (&buffer, 0); | ||
2437 | + | ||
2438 | + if (__glibc_unlikely (err == ETIMEDOUT)) | ||
2439 | + { | ||
2440 | + __condvar_dec_grefs (cond, g, private); | ||
2441 | + /* If we timed out, we effectively cancel waiting. Note that | ||
2442 | + we have decremented __g_refs before cancellation, so that a | ||
2443 | + deadlock between waiting for quiescence of our group in | ||
2444 | + __condvar_quiesce_and_switch_g1 and us trying to acquire | ||
2445 | + the lock during cancellation is not possible. */ | ||
2446 | + __condvar_cancel_waiting (cond, seq, g, private); | ||
2447 | + result = ETIMEDOUT; | ||
2448 | + goto done; | ||
2449 | + } | ||
2450 | + else | ||
2451 | + __condvar_dec_grefs (cond, g, private); | ||
2452 | + | ||
2453 | + /* Reload signals. See above for MO. */ | ||
2454 | + signals = atomic_load_acquire (cond->__data.__g_signals + g); | ||
2455 | } | ||
2456 | - pi_flag = USE_REQUEUE_PI (mutex); | ||
2457 | |||
2458 | - if (pi_flag) | ||
2459 | + } | ||
2460 | + /* Try to grab a signal. Use acquire MO so that we see an up-to-date value | ||
2461 | + of __g1_start below (see spinning above for a similar case). In | ||
2462 | + particular, if we steal from a more recent group, we will also see a | ||
2463 | + more recent __g1_start below. */ | ||
2464 | + while (!atomic_compare_exchange_weak_acquire (cond->__data.__g_signals + g, | ||
2465 | + &signals, signals - 2)); | ||
2466 | + | ||
2467 | + /* We consumed a signal but we could have consumed from a more recent group | ||
2468 | + that aliased with ours due to being in the same group slot. If this | ||
2469 | + might be the case our group must be closed as visible through | ||
2470 | + __g1_start. */ | ||
2471 | + uint64_t g1_start = __condvar_load_g1_start_relaxed (cond); | ||
2472 | + if (seq < (g1_start >> 1)) | ||
2473 | + { | ||
2474 | + /* We potentially stole a signal from a more recent group but we do not | ||
2475 | + know which group we really consumed from. | ||
2476 | + We do not care about groups older than current G1 because they are | ||
2477 | + closed; we could have stolen from these, but then we just add a | ||
2478 | + spurious wake-up for the current groups. | ||
2479 | + We will never steal a signal from current G2 that was really intended | ||
2480 | + for G2 because G2 never receives signals (until it becomes G1). We | ||
2481 | + could have stolen a signal from G2 that was conservatively added by a | ||
2482 | + previous waiter that also thought it stole a signal -- but given that | ||
2483 | + that signal was added unnecessarily, it's not a problem if we steal | ||
2484 | + it. | ||
2485 | + Thus, the remaining case is that we could have stolen from the current | ||
2486 | + G1, where "current" means the __g1_start value we observed. However, | ||
2487 | + if the current G1 does not have the same slot index as we do, we did | ||
2488 | + not steal from it and do not need to undo that. This is the reason | ||
2489 | + for putting a bit with G2's index into__g1_start as well. */ | ||
2490 | + if (((g1_start & 1) ^ 1) == g) | ||
2491 | { | ||
2492 | - err = lll_futex_wait_requeue_pi (&cond->__data.__futex, | ||
2493 | - futex_val, &mutex->__data.__lock, | ||
2494 | - pshared); | ||
2495 | - | ||
2496 | - pi_flag = (err == 0); | ||
2497 | + /* We have to conservatively undo our potential mistake of stealing | ||
2498 | + a signal. We can stop trying to do that when the current G1 | ||
2499 | + changes because other spinning waiters will notice this too and | ||
2500 | + __condvar_quiesce_and_switch_g1 has checked that there are no | ||
2501 | + futex waiters anymore before switching G1. | ||
2502 | + Relaxed MO is fine for the __g1_start load because we need to | ||
2503 | + merely be able to observe this fact and not have to observe | ||
2504 | + something else as well. | ||
2505 | + ??? Would it help to spin for a little while to see whether the | ||
2506 | + current G1 gets closed? This might be worthwhile if the group is | ||
2507 | + small or close to being closed. */ | ||
2508 | + unsigned int s = atomic_load_relaxed (cond->__data.__g_signals + g); | ||
2509 | + while (__condvar_load_g1_start_relaxed (cond) == g1_start) | ||
2510 | + { | ||
2511 | + /* Try to add a signal. We don't need to acquire the lock | ||
2512 | + because at worst we can cause a spurious wake-up. If the | ||
2513 | + group is in the process of being closed (LSB is true), this | ||
2514 | + has an effect similar to us adding a signal. */ | ||
2515 | + if (((s & 1) != 0) | ||
2516 | + || atomic_compare_exchange_weak_relaxed | ||
2517 | + (cond->__data.__g_signals + g, &s, s + 2)) | ||
2518 | + { | ||
2519 | + /* If we added a signal, we also need to add a wake-up on | ||
2520 | + the futex. We also need to do that if we skipped adding | ||
2521 | + a signal because the group is being closed because | ||
2522 | + while __condvar_quiesce_and_switch_g1 could have closed | ||
2523 | + the group, it might stil be waiting for futex waiters to | ||
2524 | + leave (and one of those waiters might be the one we stole | ||
2525 | + the signal from, which cause it to block using the | ||
2526 | + futex). */ | ||
2527 | + futex_wake (cond->__data.__g_signals + g, 1, private); | ||
2528 | + break; | ||
2529 | + } | ||
2530 | + /* TODO Back off. */ | ||
2531 | + } | ||
2532 | } | ||
2533 | - else | ||
2534 | -#endif | ||
2535 | - /* Wait until woken by signal or broadcast. */ | ||
2536 | - lll_futex_wait (&cond->__data.__futex, futex_val, pshared); | ||
2537 | - | ||
2538 | - /* Disable asynchronous cancellation. */ | ||
2539 | - __pthread_disable_asynccancel (cbuffer.oldtype); | ||
2540 | - | ||
2541 | - /* We are going to look at shared data again, so get the lock. */ | ||
2542 | - lll_lock (cond->__data.__lock, pshared); | ||
2543 | - | ||
2544 | - /* If a broadcast happened, we are done. */ | ||
2545 | - if (cbuffer.bc_seq != cond->__data.__broadcast_seq) | ||
2546 | - goto bc_out; | ||
2547 | - | ||
2548 | - /* Check whether we are eligible for wakeup. */ | ||
2549 | - val = cond->__data.__wakeup_seq; | ||
2550 | } | ||
2551 | - while (val == seq || cond->__data.__woken_seq == val); | ||
2552 | |||
2553 | - /* Another thread woken up. */ | ||
2554 | - ++cond->__data.__woken_seq; | ||
2555 | + done: | ||
2556 | |||
2557 | - bc_out: | ||
2558 | + /* Confirm that we have been woken. We do that before acquiring the mutex | ||
2559 | + to allow for execution of pthread_cond_destroy while having acquired the | ||
2560 | + mutex. */ | ||
2561 | + __condvar_confirm_wakeup (cond, private); | ||
2562 | |||
2563 | - cond->__data.__nwaiters -= 1 << COND_NWAITERS_SHIFT; | ||
2564 | - | ||
2565 | - /* If pthread_cond_destroy was called on this varaible already, | ||
2566 | - notify the pthread_cond_destroy caller all waiters have left | ||
2567 | - and it can be successfully destroyed. */ | ||
2568 | - if (cond->__data.__total_seq == -1ULL | ||
2569 | - && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT)) | ||
2570 | - lll_futex_wake (&cond->__data.__nwaiters, 1, pshared); | ||
2571 | + /* Woken up; now re-acquire the mutex. If this doesn't fail, return RESULT, | ||
2572 | + which is set to ETIMEDOUT if a timeout occured, or zero otherwise. */ | ||
2573 | + err = __pthread_mutex_cond_lock (mutex); | ||
2574 | + /* XXX Abort on errors that are disallowed by POSIX? */ | ||
2575 | + return (err != 0) ? err : result; | ||
2576 | +} | ||
2577 | |||
2578 | - /* We are done with the condvar. */ | ||
2579 | - lll_unlock (cond->__data.__lock, pshared); | ||
2580 | |||
2581 | - /* The cancellation handling is back to normal, remove the handler. */ | ||
2582 | - __pthread_cleanup_pop (&buffer, 0); | ||
2583 | +/* See __pthread_cond_wait_common. */ | ||
2584 | +int | ||
2585 | +__pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) | ||
2586 | +{ | ||
2587 | + return __pthread_cond_wait_common (cond, mutex, NULL); | ||
2588 | +} | ||
2589 | |||
2590 | - /* Get the mutex before returning. Not needed for PI. */ | ||
2591 | -#if (defined lll_futex_wait_requeue_pi \ | ||
2592 | - && defined __ASSUME_REQUEUE_PI) | ||
2593 | - if (pi_flag) | ||
2594 | - { | ||
2595 | - __pthread_mutex_cond_lock_adjust (mutex); | ||
2596 | - return 0; | ||
2597 | - } | ||
2598 | - else | ||
2599 | -#endif | ||
2600 | - return __pthread_mutex_cond_lock (mutex); | ||
2601 | +/* See __pthread_cond_wait_common. */ | ||
2602 | +int | ||
2603 | +__pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, | ||
2604 | + const struct timespec *abstime) | ||
2605 | +{ | ||
2606 | + /* Check parameter validity. This should also tell the compiler that | ||
2607 | + it can assume that abstime is not NULL. */ | ||
2608 | + if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) | ||
2609 | + return EINVAL; | ||
2610 | + return __pthread_cond_wait_common (cond, mutex, abstime); | ||
2611 | } | ||
2612 | |||
2613 | versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, | ||
2614 | GLIBC_2_3_2); | ||
2615 | +versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, | ||
2616 | + GLIBC_2_3_2); | ||
2617 | diff --git a/nptl/pthread_condattr_getclock.c b/nptl/pthread_condattr_getclock.c | ||
2618 | index d156302..cecb4aa 100644 | ||
2619 | --- a/nptl/pthread_condattr_getclock.c | ||
2620 | +++ b/nptl/pthread_condattr_getclock.c | ||
2621 | @@ -23,6 +23,6 @@ int | ||
2622 | pthread_condattr_getclock (const pthread_condattr_t *attr, clockid_t *clock_id) | ||
2623 | { | ||
2624 | *clock_id = (((((const struct pthread_condattr *) attr)->value) >> 1) | ||
2625 | - & ((1 << COND_NWAITERS_SHIFT) - 1)); | ||
2626 | + & ((1 << COND_CLOCK_BITS) - 1)); | ||
2627 | return 0; | ||
2628 | } | ||
2629 | diff --git a/nptl/pthread_condattr_getpshared.c b/nptl/pthread_condattr_getpshared.c | ||
2630 | index 5a10f3e..8147966 100644 | ||
2631 | --- a/nptl/pthread_condattr_getpshared.c | ||
2632 | +++ b/nptl/pthread_condattr_getpshared.c | ||
2633 | @@ -22,7 +22,8 @@ | ||
2634 | int | ||
2635 | pthread_condattr_getpshared (const pthread_condattr_t *attr, int *pshared) | ||
2636 | { | ||
2637 | - *pshared = ((const struct pthread_condattr *) attr)->value & 1; | ||
2638 | + *pshared = (((const struct pthread_condattr *) attr)->value & 1 | ||
2639 | + ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE); | ||
2640 | |||
2641 | return 0; | ||
2642 | } | ||
2643 | diff --git a/nptl/pthread_condattr_init.c b/nptl/pthread_condattr_init.c | ||
2644 | index 0ce42e5..6e5168d 100644 | ||
2645 | --- a/nptl/pthread_condattr_init.c | ||
2646 | +++ b/nptl/pthread_condattr_init.c | ||
2647 | @@ -23,7 +23,9 @@ | ||
2648 | int | ||
2649 | __pthread_condattr_init (pthread_condattr_t *attr) | ||
2650 | { | ||
2651 | - memset (attr, '\0', sizeof (*attr)); | ||
2652 | + struct pthread_condattr *iattr = (struct pthread_condattr *) attr; | ||
2653 | + /* Default is not pshared and CLOCK_REALTIME. */ | ||
2654 | + iattr-> value = CLOCK_REALTIME << 1; | ||
2655 | |||
2656 | return 0; | ||
2657 | } | ||
2658 | diff --git a/nptl/pthread_condattr_setclock.c b/nptl/pthread_condattr_setclock.c | ||
2659 | index 25e2a17..3cfad84 100644 | ||
2660 | --- a/nptl/pthread_condattr_setclock.c | ||
2661 | +++ b/nptl/pthread_condattr_setclock.c | ||
2662 | @@ -18,7 +18,7 @@ | ||
2663 | |||
2664 | #include <assert.h> | ||
2665 | #include <errno.h> | ||
2666 | -#include <stdbool.h> | ||
2667 | +#include <futex-internal.h> | ||
2668 | #include <time.h> | ||
2669 | #include <sysdep.h> | ||
2670 | #include "pthreadP.h" | ||
2671 | @@ -33,12 +33,17 @@ pthread_condattr_setclock (pthread_condattr_t *attr, clockid_t clock_id) | ||
2672 | in the pthread_cond_t structure needs to be adjusted. */ | ||
2673 | return EINVAL; | ||
2674 | |||
2675 | + /* If we do not support waiting using CLOCK_MONOTONIC, return an error. */ | ||
2676 | + if (clock_id == CLOCK_MONOTONIC | ||
2677 | + && !futex_supports_exact_relative_timeouts()) | ||
2678 | + return ENOTSUP; | ||
2679 | + | ||
2680 | /* Make sure the value fits in the bits we reserved. */ | ||
2681 | - assert (clock_id < (1 << COND_NWAITERS_SHIFT)); | ||
2682 | + assert (clock_id < (1 << COND_CLOCK_BITS)); | ||
2683 | |||
2684 | int *valuep = &((struct pthread_condattr *) attr)->value; | ||
2685 | |||
2686 | - *valuep = ((*valuep & ~(((1 << COND_NWAITERS_SHIFT) - 1) << 1)) | ||
2687 | + *valuep = ((*valuep & ~(((1 << COND_CLOCK_BITS) - 1) << 1)) | ||
2688 | | (clock_id << 1)); | ||
2689 | |||
2690 | return 0; | ||
2691 | diff --git a/nptl/test-cond-printers.py b/nptl/test-cond-printers.py | ||
2692 | index af0e12e..9e807c9 100644 | ||
2693 | --- a/nptl/test-cond-printers.py | ||
2694 | +++ b/nptl/test-cond-printers.py | ||
2695 | @@ -35,7 +35,7 @@ try: | ||
2696 | |||
2697 | break_at(test_source, 'Test status (destroyed)') | ||
2698 | continue_cmd() # Go to test_status_destroyed | ||
2699 | - test_printer(var, to_string, {'Status': 'Destroyed'}) | ||
2700 | + test_printer(var, to_string, {'Threads known to still execute a wait function': '0'}) | ||
2701 | |||
2702 | continue_cmd() # Exit | ||
2703 | |||
2704 | diff --git a/nptl/tst-cond1.c b/nptl/tst-cond1.c | ||
2705 | index 75ab9c8..509bbd0 100644 | ||
2706 | --- a/nptl/tst-cond1.c | ||
2707 | +++ b/nptl/tst-cond1.c | ||
2708 | @@ -73,6 +73,9 @@ do_test (void) | ||
2709 | |||
2710 | puts ("parent: wait for condition"); | ||
2711 | |||
2712 | + /* This test will fail on spurious wake-ups, which are allowed; however, | ||
2713 | + the current implementation shouldn't produce spurious wake-ups in the | ||
2714 | + scenario we are testing here. */ | ||
2715 | err = pthread_cond_wait (&cond, &mut); | ||
2716 | if (err != 0) | ||
2717 | error (EXIT_FAILURE, err, "parent: cannot wait fir signal"); | ||
2718 | diff --git a/nptl/tst-cond20.c b/nptl/tst-cond20.c | ||
2719 | index 918c4ad..665a66a 100644 | ||
2720 | --- a/nptl/tst-cond20.c | ||
2721 | +++ b/nptl/tst-cond20.c | ||
2722 | @@ -96,7 +96,10 @@ do_test (void) | ||
2723 | |||
2724 | for (i = 0; i < ROUNDS; ++i) | ||
2725 | { | ||
2726 | - pthread_cond_wait (&cond2, &mut); | ||
2727 | + /* Make sure we discard spurious wake-ups. */ | ||
2728 | + do | ||
2729 | + pthread_cond_wait (&cond2, &mut); | ||
2730 | + while (count != N); | ||
2731 | |||
2732 | if (i & 1) | ||
2733 | pthread_mutex_unlock (&mut); | ||
2734 | diff --git a/nptl/tst-cond22.c b/nptl/tst-cond22.c | ||
2735 | index bd978e5..64f19ea 100644 | ||
2736 | --- a/nptl/tst-cond22.c | ||
2737 | +++ b/nptl/tst-cond22.c | ||
2738 | @@ -106,10 +106,11 @@ do_test (void) | ||
2739 | status = 1; | ||
2740 | } | ||
2741 | |||
2742 | - printf ("cond = { %d, %x, %lld, %lld, %lld, %p, %u, %u }\n", | ||
2743 | - c.__data.__lock, c.__data.__futex, c.__data.__total_seq, | ||
2744 | - c.__data.__wakeup_seq, c.__data.__woken_seq, c.__data.__mutex, | ||
2745 | - c.__data.__nwaiters, c.__data.__broadcast_seq); | ||
2746 | + printf ("cond = { %llu, %llu, %u/%u/%u, %u/%u/%u, %u, %u }\n", | ||
2747 | + c.__data.__wseq, c.__data.__g1_start, | ||
2748 | + c.__data.__g_signals[0], c.__data.__g_refs[0], c.__data.__g_size[0], | ||
2749 | + c.__data.__g_signals[1], c.__data.__g_refs[1], c.__data.__g_size[1], | ||
2750 | + c.__data.__g1_orig_size, c.__data.__wrefs); | ||
2751 | |||
2752 | if (pthread_create (&th, NULL, tf, (void *) 1l) != 0) | ||
2753 | { | ||
2754 | @@ -148,10 +149,11 @@ do_test (void) | ||
2755 | status = 1; | ||
2756 | } | ||
2757 | |||
2758 | - printf ("cond = { %d, %x, %lld, %lld, %lld, %p, %u, %u }\n", | ||
2759 | - c.__data.__lock, c.__data.__futex, c.__data.__total_seq, | ||
2760 | - c.__data.__wakeup_seq, c.__data.__woken_seq, c.__data.__mutex, | ||
2761 | - c.__data.__nwaiters, c.__data.__broadcast_seq); | ||
2762 | + printf ("cond = { %llu, %llu, %u/%u/%u, %u/%u/%u, %u, %u }\n", | ||
2763 | + c.__data.__wseq, c.__data.__g1_start, | ||
2764 | + c.__data.__g_signals[0], c.__data.__g_refs[0], c.__data.__g_size[0], | ||
2765 | + c.__data.__g_signals[1], c.__data.__g_refs[1], c.__data.__g_size[1], | ||
2766 | + c.__data.__g1_orig_size, c.__data.__wrefs); | ||
2767 | |||
2768 | return status; | ||
2769 | } | ||
2770 | diff --git a/sysdeps/aarch64/nptl/bits/pthreadtypes.h b/sysdeps/aarch64/nptl/bits/pthreadtypes.h | ||
2771 | index 13984a7..c6fa632 100644 | ||
2772 | --- a/sysdeps/aarch64/nptl/bits/pthreadtypes.h | ||
2773 | +++ b/sysdeps/aarch64/nptl/bits/pthreadtypes.h | ||
2774 | @@ -90,17 +90,30 @@ typedef union | ||
2775 | { | ||
2776 | struct | ||
2777 | { | ||
2778 | - int __lock; | ||
2779 | - unsigned int __futex; | ||
2780 | - __extension__ unsigned long long int __total_seq; | ||
2781 | - __extension__ unsigned long long int __wakeup_seq; | ||
2782 | - __extension__ unsigned long long int __woken_seq; | ||
2783 | - void *__mutex; | ||
2784 | - unsigned int __nwaiters; | ||
2785 | - unsigned int __broadcast_seq; | ||
2786 | + __extension__ union | ||
2787 | + { | ||
2788 | + __extension__ unsigned long long int __wseq; | ||
2789 | + struct { | ||
2790 | + unsigned int __low; | ||
2791 | + unsigned int __high; | ||
2792 | + } __wseq32; | ||
2793 | + }; | ||
2794 | + __extension__ union | ||
2795 | + { | ||
2796 | + __extension__ unsigned long long int __g1_start; | ||
2797 | + struct { | ||
2798 | + unsigned int __low; | ||
2799 | + unsigned int __high; | ||
2800 | + } __g1_start32; | ||
2801 | + }; | ||
2802 | + unsigned int __g_refs[2]; | ||
2803 | + unsigned int __g_size[2]; | ||
2804 | + unsigned int __g1_orig_size; | ||
2805 | + unsigned int __wrefs; | ||
2806 | + unsigned int __g_signals[2]; | ||
2807 | } __data; | ||
2808 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
2809 | - long int __align; | ||
2810 | + __extension__ long long int __align; | ||
2811 | } pthread_cond_t; | ||
2812 | |||
2813 | typedef union | ||
2814 | diff --git a/sysdeps/arm/nptl/bits/pthreadtypes.h b/sysdeps/arm/nptl/bits/pthreadtypes.h | ||
2815 | index afb5392..53518c6 100644 | ||
2816 | --- a/sysdeps/arm/nptl/bits/pthreadtypes.h | ||
2817 | +++ b/sysdeps/arm/nptl/bits/pthreadtypes.h | ||
2818 | @@ -93,14 +93,27 @@ typedef union | ||
2819 | { | ||
2820 | struct | ||
2821 | { | ||
2822 | - int __lock; | ||
2823 | - unsigned int __futex; | ||
2824 | - __extension__ unsigned long long int __total_seq; | ||
2825 | - __extension__ unsigned long long int __wakeup_seq; | ||
2826 | - __extension__ unsigned long long int __woken_seq; | ||
2827 | - void *__mutex; | ||
2828 | - unsigned int __nwaiters; | ||
2829 | - unsigned int __broadcast_seq; | ||
2830 | + __extension__ union | ||
2831 | + { | ||
2832 | + __extension__ unsigned long long int __wseq; | ||
2833 | + struct { | ||
2834 | + unsigned int __low; | ||
2835 | + unsigned int __high; | ||
2836 | + } __wseq32; | ||
2837 | + }; | ||
2838 | + __extension__ union | ||
2839 | + { | ||
2840 | + __extension__ unsigned long long int __g1_start; | ||
2841 | + struct { | ||
2842 | + unsigned int __low; | ||
2843 | + unsigned int __high; | ||
2844 | + } __g1_start32; | ||
2845 | + }; | ||
2846 | + unsigned int __g_refs[2]; | ||
2847 | + unsigned int __g_size[2]; | ||
2848 | + unsigned int __g1_orig_size; | ||
2849 | + unsigned int __wrefs; | ||
2850 | + unsigned int __g_signals[2]; | ||
2851 | } __data; | ||
2852 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
2853 | __extension__ long long int __align; | ||
2854 | diff --git a/sysdeps/ia64/nptl/bits/pthreadtypes.h b/sysdeps/ia64/nptl/bits/pthreadtypes.h | ||
2855 | index f2e6dac..e72dbfd 100644 | ||
2856 | --- a/sysdeps/ia64/nptl/bits/pthreadtypes.h | ||
2857 | +++ b/sysdeps/ia64/nptl/bits/pthreadtypes.h | ||
2858 | @@ -90,17 +90,30 @@ typedef union | ||
2859 | { | ||
2860 | struct | ||
2861 | { | ||
2862 | - int __lock; | ||
2863 | - unsigned int __futex; | ||
2864 | - __extension__ unsigned long long int __total_seq; | ||
2865 | - __extension__ unsigned long long int __wakeup_seq; | ||
2866 | - __extension__ unsigned long long int __woken_seq; | ||
2867 | - void *__mutex; | ||
2868 | - unsigned int __nwaiters; | ||
2869 | - unsigned int __broadcast_seq; | ||
2870 | + __extension__ union | ||
2871 | + { | ||
2872 | + __extension__ unsigned long long int __wseq; | ||
2873 | + struct { | ||
2874 | + unsigned int __low; | ||
2875 | + unsigned int __high; | ||
2876 | + } __wseq32; | ||
2877 | + }; | ||
2878 | + __extension__ union | ||
2879 | + { | ||
2880 | + __extension__ unsigned long long int __g1_start; | ||
2881 | + struct { | ||
2882 | + unsigned int __low; | ||
2883 | + unsigned int __high; | ||
2884 | + } __g1_start32; | ||
2885 | + }; | ||
2886 | + unsigned int __g_refs[2]; | ||
2887 | + unsigned int __g_size[2]; | ||
2888 | + unsigned int __g1_orig_size; | ||
2889 | + unsigned int __wrefs; | ||
2890 | + unsigned int __g_signals[2]; | ||
2891 | } __data; | ||
2892 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
2893 | - long int __align; | ||
2894 | + __extension__ long long int __align; | ||
2895 | } pthread_cond_t; | ||
2896 | |||
2897 | typedef union | ||
2898 | diff --git a/sysdeps/m68k/nptl/bits/pthreadtypes.h b/sysdeps/m68k/nptl/bits/pthreadtypes.h | ||
2899 | index d8faa7a..c5e9021 100644 | ||
2900 | --- a/sysdeps/m68k/nptl/bits/pthreadtypes.h | ||
2901 | +++ b/sysdeps/m68k/nptl/bits/pthreadtypes.h | ||
2902 | @@ -88,19 +88,33 @@ typedef union | ||
2903 | |||
2904 | |||
2905 | /* Data structure for conditional variable handling. The structure of | ||
2906 | - the attribute type is deliberately not exposed. */ | ||
2907 | + the attribute type is not exposed on purpose. */ | ||
2908 | typedef union | ||
2909 | { | ||
2910 | struct | ||
2911 | { | ||
2912 | - int __lock __attribute__ ((__aligned__ (4))); | ||
2913 | - unsigned int __futex; | ||
2914 | - __extension__ unsigned long long int __total_seq; | ||
2915 | - __extension__ unsigned long long int __wakeup_seq; | ||
2916 | - __extension__ unsigned long long int __woken_seq; | ||
2917 | - void *__mutex; | ||
2918 | - unsigned int __nwaiters; | ||
2919 | - unsigned int __broadcast_seq; | ||
2920 | + __extension__ union | ||
2921 | + { | ||
2922 | + __extension__ unsigned long long int __wseq; | ||
2923 | + struct { | ||
2924 | + unsigned int __low; | ||
2925 | + unsigned int __high; | ||
2926 | + } __wseq32; | ||
2927 | + }; | ||
2928 | + __extension__ union | ||
2929 | + { | ||
2930 | + __extension__ unsigned long long int __g1_start; | ||
2931 | + struct { | ||
2932 | + unsigned int __low; | ||
2933 | + unsigned int __high; | ||
2934 | + } __g1_start32; | ||
2935 | + }; | ||
2936 | + /* Enforce proper alignment of fields used as futex words. */ | ||
2937 | + unsigned int __g_refs[2] __attribute__ ((__aligned__ (4))); | ||
2938 | + unsigned int __g_size[2]; | ||
2939 | + unsigned int __g1_orig_size; | ||
2940 | + unsigned int __wrefs; | ||
2941 | + unsigned int __g_signals[2]; | ||
2942 | } __data; | ||
2943 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
2944 | __extension__ long long int __align; | ||
2945 | diff --git a/sysdeps/microblaze/nptl/bits/pthreadtypes.h b/sysdeps/microblaze/nptl/bits/pthreadtypes.h | ||
2946 | index 9e9e307..b6623c2 100644 | ||
2947 | --- a/sysdeps/microblaze/nptl/bits/pthreadtypes.h | ||
2948 | +++ b/sysdeps/microblaze/nptl/bits/pthreadtypes.h | ||
2949 | @@ -91,14 +91,27 @@ typedef union | ||
2950 | { | ||
2951 | struct | ||
2952 | { | ||
2953 | - int __lock; | ||
2954 | - unsigned int __futex; | ||
2955 | - __extension__ unsigned long long int __total_seq; | ||
2956 | - __extension__ unsigned long long int __wakeup_seq; | ||
2957 | - __extension__ unsigned long long int __woken_seq; | ||
2958 | - void *__mutex; | ||
2959 | - unsigned int __nwaiters; | ||
2960 | - unsigned int __broadcast_seq; | ||
2961 | + __extension__ union | ||
2962 | + { | ||
2963 | + __extension__ unsigned long long int __wseq; | ||
2964 | + struct { | ||
2965 | + unsigned int __low; | ||
2966 | + unsigned int __high; | ||
2967 | + } __wseq32; | ||
2968 | + }; | ||
2969 | + __extension__ union | ||
2970 | + { | ||
2971 | + __extension__ unsigned long long int __g1_start; | ||
2972 | + struct { | ||
2973 | + unsigned int __low; | ||
2974 | + unsigned int __high; | ||
2975 | + } __g1_start32; | ||
2976 | + }; | ||
2977 | + unsigned int __g_refs[2]; | ||
2978 | + unsigned int __g_size[2]; | ||
2979 | + unsigned int __g1_orig_size; | ||
2980 | + unsigned int __wrefs; | ||
2981 | + unsigned int __g_signals[2]; | ||
2982 | } __data; | ||
2983 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
2984 | __extension__ long long int __align; | ||
2985 | diff --git a/sysdeps/mips/nptl/bits/pthreadtypes.h b/sysdeps/mips/nptl/bits/pthreadtypes.h | ||
2986 | index 68ed94b..7ddc7bf 100644 | ||
2987 | --- a/sysdeps/mips/nptl/bits/pthreadtypes.h | ||
2988 | +++ b/sysdeps/mips/nptl/bits/pthreadtypes.h | ||
2989 | @@ -117,19 +117,32 @@ typedef union | ||
2990 | |||
2991 | |||
2992 | /* Data structure for conditional variable handling. The structure of | ||
2993 | - the attribute type is deliberately not exposed. */ | ||
2994 | + the attribute type is not exposed on purpose. */ | ||
2995 | typedef union | ||
2996 | { | ||
2997 | struct | ||
2998 | { | ||
2999 | - int __lock; | ||
3000 | - unsigned int __futex; | ||
3001 | - __extension__ unsigned long long int __total_seq; | ||
3002 | - __extension__ unsigned long long int __wakeup_seq; | ||
3003 | - __extension__ unsigned long long int __woken_seq; | ||
3004 | - void *__mutex; | ||
3005 | - unsigned int __nwaiters; | ||
3006 | - unsigned int __broadcast_seq; | ||
3007 | + __extension__ union | ||
3008 | + { | ||
3009 | + __extension__ unsigned long long int __wseq; | ||
3010 | + struct { | ||
3011 | + unsigned int __low; | ||
3012 | + unsigned int __high; | ||
3013 | + } __wseq32; | ||
3014 | + }; | ||
3015 | + __extension__ union | ||
3016 | + { | ||
3017 | + __extension__ unsigned long long int __g1_start; | ||
3018 | + struct { | ||
3019 | + unsigned int __low; | ||
3020 | + unsigned int __high; | ||
3021 | + } __g1_start32; | ||
3022 | + }; | ||
3023 | + unsigned int __g_refs[2]; | ||
3024 | + unsigned int __g_size[2]; | ||
3025 | + unsigned int __g1_orig_size; | ||
3026 | + unsigned int __wrefs; | ||
3027 | + unsigned int __g_signals[2]; | ||
3028 | } __data; | ||
3029 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
3030 | __extension__ long long int __align; | ||
3031 | diff --git a/sysdeps/nios2/nptl/bits/pthreadtypes.h b/sysdeps/nios2/nptl/bits/pthreadtypes.h | ||
3032 | index 76076d0..3995e26 100644 | ||
3033 | --- a/sysdeps/nios2/nptl/bits/pthreadtypes.h | ||
3034 | +++ b/sysdeps/nios2/nptl/bits/pthreadtypes.h | ||
3035 | @@ -88,19 +88,32 @@ typedef union | ||
3036 | |||
3037 | |||
3038 | /* Data structure for conditional variable handling. The structure of | ||
3039 | - the attribute type is deliberately not exposed. */ | ||
3040 | + the attribute type is not exposed on purpose. */ | ||
3041 | typedef union | ||
3042 | { | ||
3043 | struct | ||
3044 | { | ||
3045 | - int __lock; | ||
3046 | - unsigned int __futex; | ||
3047 | - __extension__ unsigned long long int __total_seq; | ||
3048 | - __extension__ unsigned long long int __wakeup_seq; | ||
3049 | - __extension__ unsigned long long int __woken_seq; | ||
3050 | - void *__mutex; | ||
3051 | - unsigned int __nwaiters; | ||
3052 | - unsigned int __broadcast_seq; | ||
3053 | + __extension__ union | ||
3054 | + { | ||
3055 | + __extension__ unsigned long long int __wseq; | ||
3056 | + struct { | ||
3057 | + unsigned int __low; | ||
3058 | + unsigned int __high; | ||
3059 | + } __wseq32; | ||
3060 | + }; | ||
3061 | + __extension__ union | ||
3062 | + { | ||
3063 | + __extension__ unsigned long long int __g1_start; | ||
3064 | + struct { | ||
3065 | + unsigned int __low; | ||
3066 | + unsigned int __high; | ||
3067 | + } __g1_start32; | ||
3068 | + }; | ||
3069 | + unsigned int __g_refs[2]; | ||
3070 | + unsigned int __g_size[2]; | ||
3071 | + unsigned int __g1_orig_size; | ||
3072 | + unsigned int __wrefs; | ||
3073 | + unsigned int __g_signals[2]; | ||
3074 | } __data; | ||
3075 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
3076 | __extension__ long long int __align; | ||
3077 | diff --git a/sysdeps/nptl/internaltypes.h b/sysdeps/nptl/internaltypes.h | ||
3078 | index 203c548..31e5a43 100644 | ||
3079 | --- a/sysdeps/nptl/internaltypes.h | ||
3080 | +++ b/sysdeps/nptl/internaltypes.h | ||
3081 | @@ -68,20 +68,13 @@ struct pthread_condattr | ||
3082 | { | ||
3083 | /* Combination of values: | ||
3084 | |||
3085 | - Bit 0 : flag whether conditional variable will be sharable between | ||
3086 | - processes. | ||
3087 | - | ||
3088 | - Bit 1-7: clock ID. */ | ||
3089 | + Bit 0 : flag whether conditional variable will be | ||
3090 | + sharable between processes. | ||
3091 | + Bit 1-COND_CLOCK_BITS: Clock ID. COND_CLOCK_BITS is the number of bits | ||
3092 | + needed to represent the ID of the clock. */ | ||
3093 | int value; | ||
3094 | }; | ||
3095 | - | ||
3096 | - | ||
3097 | -/* The __NWAITERS field is used as a counter and to house the number | ||
3098 | - of bits for other purposes. COND_CLOCK_BITS is the number | ||
3099 | - of bits needed to represent the ID of the clock. COND_NWAITERS_SHIFT | ||
3100 | - is the number of bits reserved for other purposes like the clock. */ | ||
3101 | -#define COND_CLOCK_BITS 1 | ||
3102 | -#define COND_NWAITERS_SHIFT 1 | ||
3103 | +#define COND_CLOCK_BITS 1 | ||
3104 | |||
3105 | |||
3106 | /* Read-write lock variable attribute data structure. */ | ||
3107 | diff --git a/sysdeps/nptl/pthread.h b/sysdeps/nptl/pthread.h | ||
3108 | index fd0894e..c122446 100644 | ||
3109 | --- a/sysdeps/nptl/pthread.h | ||
3110 | +++ b/sysdeps/nptl/pthread.h | ||
3111 | @@ -183,7 +183,7 @@ enum | ||
3112 | |||
3113 | |||
3114 | /* Conditional variable handling. */ | ||
3115 | -#define PTHREAD_COND_INITIALIZER { { 0, 0, 0, 0, 0, (void *) 0, 0, 0 } } | ||
3116 | +#define PTHREAD_COND_INITIALIZER { { {0}, {0}, {0, 0}, {0, 0}, 0, 0, {0, 0} } } | ||
3117 | |||
3118 | |||
3119 | /* Cleanup buffers */ | ||
3120 | diff --git a/sysdeps/s390/nptl/bits/pthreadtypes.h b/sysdeps/s390/nptl/bits/pthreadtypes.h | ||
3121 | index 40d10fe..4e455ab 100644 | ||
3122 | --- a/sysdeps/s390/nptl/bits/pthreadtypes.h | ||
3123 | +++ b/sysdeps/s390/nptl/bits/pthreadtypes.h | ||
3124 | @@ -142,14 +142,27 @@ typedef union | ||
3125 | { | ||
3126 | struct | ||
3127 | { | ||
3128 | - int __lock; | ||
3129 | - unsigned int __futex; | ||
3130 | - __extension__ unsigned long long int __total_seq; | ||
3131 | - __extension__ unsigned long long int __wakeup_seq; | ||
3132 | - __extension__ unsigned long long int __woken_seq; | ||
3133 | - void *__mutex; | ||
3134 | - unsigned int __nwaiters; | ||
3135 | - unsigned int __broadcast_seq; | ||
3136 | + __extension__ union | ||
3137 | + { | ||
3138 | + __extension__ unsigned long long int __wseq; | ||
3139 | + struct { | ||
3140 | + unsigned int __low; | ||
3141 | + unsigned int __high; | ||
3142 | + } __wseq32; | ||
3143 | + }; | ||
3144 | + __extension__ union | ||
3145 | + { | ||
3146 | + __extension__ unsigned long long int __g1_start; | ||
3147 | + struct { | ||
3148 | + unsigned int __low; | ||
3149 | + unsigned int __high; | ||
3150 | + } __g1_start32; | ||
3151 | + }; | ||
3152 | + unsigned int __g_refs[2]; | ||
3153 | + unsigned int __g_size[2]; | ||
3154 | + unsigned int __g1_orig_size; | ||
3155 | + unsigned int __wrefs; | ||
3156 | + unsigned int __g_signals[2]; | ||
3157 | } __data; | ||
3158 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
3159 | __extension__ long long int __align; | ||
3160 | diff --git a/sysdeps/sh/nptl/bits/pthreadtypes.h b/sysdeps/sh/nptl/bits/pthreadtypes.h | ||
3161 | index 13fbd73..065dd11 100644 | ||
3162 | --- a/sysdeps/sh/nptl/bits/pthreadtypes.h | ||
3163 | +++ b/sysdeps/sh/nptl/bits/pthreadtypes.h | ||
3164 | @@ -93,14 +93,27 @@ typedef union | ||
3165 | { | ||
3166 | struct | ||
3167 | { | ||
3168 | - int __lock; | ||
3169 | - unsigned int __futex; | ||
3170 | - __extension__ unsigned long long int __total_seq; | ||
3171 | - __extension__ unsigned long long int __wakeup_seq; | ||
3172 | - __extension__ unsigned long long int __woken_seq; | ||
3173 | - void *__mutex; | ||
3174 | - unsigned int __nwaiters; | ||
3175 | - unsigned int __broadcast_seq; | ||
3176 | + __extension__ union | ||
3177 | + { | ||
3178 | + __extension__ unsigned long long int __wseq; | ||
3179 | + struct { | ||
3180 | + unsigned int __low; | ||
3181 | + unsigned int __high; | ||
3182 | + } __wseq32; | ||
3183 | + }; | ||
3184 | + __extension__ union | ||
3185 | + { | ||
3186 | + __extension__ unsigned long long int __g1_start; | ||
3187 | + struct { | ||
3188 | + unsigned int __low; | ||
3189 | + unsigned int __high; | ||
3190 | + } __g1_start32; | ||
3191 | + }; | ||
3192 | + unsigned int __g_refs[2]; | ||
3193 | + unsigned int __g_size[2]; | ||
3194 | + unsigned int __g1_orig_size; | ||
3195 | + unsigned int __wrefs; | ||
3196 | + unsigned int __g_signals[2]; | ||
3197 | } __data; | ||
3198 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
3199 | __extension__ long long int __align; | ||
3200 | diff --git a/sysdeps/tile/nptl/bits/pthreadtypes.h b/sysdeps/tile/nptl/bits/pthreadtypes.h | ||
3201 | index 7d68650..c12737f 100644 | ||
3202 | --- a/sysdeps/tile/nptl/bits/pthreadtypes.h | ||
3203 | +++ b/sysdeps/tile/nptl/bits/pthreadtypes.h | ||
3204 | @@ -122,14 +122,27 @@ typedef union | ||
3205 | { | ||
3206 | struct | ||
3207 | { | ||
3208 | - int __lock; | ||
3209 | - unsigned int __futex; | ||
3210 | - __extension__ unsigned long long int __total_seq; | ||
3211 | - __extension__ unsigned long long int __wakeup_seq; | ||
3212 | - __extension__ unsigned long long int __woken_seq; | ||
3213 | - void *__mutex; | ||
3214 | - unsigned int __nwaiters; | ||
3215 | - unsigned int __broadcast_seq; | ||
3216 | + __extension__ union | ||
3217 | + { | ||
3218 | + __extension__ unsigned long long int __wseq; | ||
3219 | + struct { | ||
3220 | + unsigned int __low; | ||
3221 | + unsigned int __high; | ||
3222 | + } __wseq32; | ||
3223 | + }; | ||
3224 | + __extension__ union | ||
3225 | + { | ||
3226 | + __extension__ unsigned long long int __g1_start; | ||
3227 | + struct { | ||
3228 | + unsigned int __low; | ||
3229 | + unsigned int __high; | ||
3230 | + } __g1_start32; | ||
3231 | + }; | ||
3232 | + unsigned int __g_refs[2]; | ||
3233 | + unsigned int __g_size[2]; | ||
3234 | + unsigned int __g1_orig_size; | ||
3235 | + unsigned int __wrefs; | ||
3236 | + unsigned int __g_signals[2]; | ||
3237 | } __data; | ||
3238 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
3239 | __extension__ long long int __align; | ||
3240 | diff --git a/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h | ||
3241 | index 1a1779b..d88b045 100644 | ||
3242 | --- a/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h | ||
3243 | +++ b/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h | ||
3244 | @@ -84,19 +84,32 @@ typedef union | ||
3245 | |||
3246 | |||
3247 | /* Data structure for conditional variable handling. The structure of | ||
3248 | - the attribute type is deliberately not exposed. */ | ||
3249 | + the attribute type is not exposed on purpose. */ | ||
3250 | typedef union | ||
3251 | { | ||
3252 | struct | ||
3253 | { | ||
3254 | - int __lock; | ||
3255 | - unsigned int __futex; | ||
3256 | - __extension__ unsigned long long int __total_seq; | ||
3257 | - __extension__ unsigned long long int __wakeup_seq; | ||
3258 | - __extension__ unsigned long long int __woken_seq; | ||
3259 | - void *__mutex; | ||
3260 | - unsigned int __nwaiters; | ||
3261 | - unsigned int __broadcast_seq; | ||
3262 | + __extension__ union | ||
3263 | + { | ||
3264 | + __extension__ unsigned long long int __wseq; | ||
3265 | + struct { | ||
3266 | + unsigned int __low; | ||
3267 | + unsigned int __high; | ||
3268 | + } __wseq32; | ||
3269 | + }; | ||
3270 | + __extension__ union | ||
3271 | + { | ||
3272 | + __extension__ unsigned long long int __g1_start; | ||
3273 | + struct { | ||
3274 | + unsigned int __low; | ||
3275 | + unsigned int __high; | ||
3276 | + } __g1_start32; | ||
3277 | + }; | ||
3278 | + unsigned int __g_refs[2]; | ||
3279 | + unsigned int __g_size[2]; | ||
3280 | + unsigned int __g1_orig_size; | ||
3281 | + unsigned int __wrefs; | ||
3282 | + unsigned int __g_signals[2]; | ||
3283 | } __data; | ||
3284 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
3285 | __extension__ long long int __align; | ||
3286 | diff --git a/sysdeps/unix/sysv/linux/hppa/internaltypes.h b/sysdeps/unix/sysv/linux/hppa/internaltypes.h | ||
3287 | index 651ce2e..d649657 100644 | ||
3288 | --- a/sysdeps/unix/sysv/linux/hppa/internaltypes.h | ||
3289 | +++ b/sysdeps/unix/sysv/linux/hppa/internaltypes.h | ||
3290 | @@ -46,32 +46,38 @@ fails because __initializer is zero, and the structure will be used as | ||
3291 | is correctly. */ | ||
3292 | |||
3293 | #define cond_compat_clear(var) \ | ||
3294 | -({ \ | ||
3295 | - int tmp = 0; \ | ||
3296 | - var->__data.__lock = 0; \ | ||
3297 | - var->__data.__futex = 0; \ | ||
3298 | - var->__data.__mutex = NULL; \ | ||
3299 | - /* Clear __initializer last, to indicate initialization is done. */ \ | ||
3300 | - __asm__ __volatile__ ("stw,ma %1,0(%0)" \ | ||
3301 | - : : "r" (&var->__data.__initializer), "r" (tmp) : "memory"); \ | ||
3302 | +({ \ | ||
3303 | + int tmp = 0; \ | ||
3304 | + var->__data.__wseq = 0; \ | ||
3305 | + var->__data.__signals_sent = 0; \ | ||
3306 | + var->__data.__confirmed = 0; \ | ||
3307 | + var->__data.__generation = 0; \ | ||
3308 | + var->__data.__mutex = NULL; \ | ||
3309 | + var->__data.__quiescence_waiters = 0; \ | ||
3310 | + var->__data.__clockid = 0; \ | ||
3311 | + /* Clear __initializer last, to indicate initialization is done. */ \ | ||
3312 | + /* This synchronizes-with the acquire load below. */ \ | ||
3313 | + atomic_store_release (&var->__data.__initializer, 0); \ | ||
3314 | }) | ||
3315 | |||
3316 | #define cond_compat_check_and_clear(var) \ | ||
3317 | ({ \ | ||
3318 | - int ret; \ | ||
3319 | - volatile int *value = &var->__data.__initializer; \ | ||
3320 | - if ((ret = atomic_compare_and_exchange_val_acq(value, 2, 1))) \ | ||
3321 | + int v; \ | ||
3322 | + int *value = &var->__data.__initializer; \ | ||
3323 | + /* This synchronizes-with the release store above. */ \ | ||
3324 | + while ((v = atomic_load_acquire (value)) != 0) \ | ||
3325 | { \ | ||
3326 | - if (ret == 1) \ | ||
3327 | + if (v == 1 \ | ||
3328 | + /* Relaxed MO is fine; it only matters who's first. */ \ | ||
3329 | + && atomic_compare_exchange_acquire_weak_relaxed (value, 1, 2)) \ | ||
3330 | { \ | ||
3331 | - /* Initialize structure. */ \ | ||
3332 | + /* We're first; initialize structure. */ \ | ||
3333 | cond_compat_clear (var); \ | ||
3334 | + break; \ | ||
3335 | } \ | ||
3336 | else \ | ||
3337 | - { \ | ||
3338 | - /* Yield until structure is initialized. */ \ | ||
3339 | - while (*value == 2) sched_yield (); \ | ||
3340 | - } \ | ||
3341 | + /* Yield before we re-check initialization status. */ \ | ||
3342 | + sched_yield (); \ | ||
3343 | } \ | ||
3344 | }) | ||
3345 | |||
3346 | diff --git a/sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c b/sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c | ||
3347 | deleted file mode 100644 | ||
3348 | index ec6fd23..0000000 | ||
3349 | --- a/sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c | ||
3350 | +++ /dev/null | ||
3351 | @@ -1,41 +0,0 @@ | ||
3352 | -/* Copyright (C) 2009-2016 Free Software Foundation, Inc. | ||
3353 | - This file is part of the GNU C Library. | ||
3354 | - Contributed by Carlos O'Donell <carlos@codesourcery.com>, 2009. | ||
3355 | - | ||
3356 | - The GNU C Library is free software; you can redistribute it and/or | ||
3357 | - modify it under the terms of the GNU Lesser General Public | ||
3358 | - License as published by the Free Software Foundation; either | ||
3359 | - version 2.1 of the License, or (at your option) any later version. | ||
3360 | - | ||
3361 | - The GNU C Library is distributed in the hope that it will be useful, | ||
3362 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3363 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
3364 | - Lesser General Public License for more details. | ||
3365 | - | ||
3366 | - You should have received a copy of the GNU Lesser General Public | ||
3367 | - License along with the GNU C Library. If not, see | ||
3368 | - <http://www.gnu.org/licenses/>. */ | ||
3369 | - | ||
3370 | -#ifndef INCLUDED_SELF | ||
3371 | -# define INCLUDED_SELF | ||
3372 | -# include <pthread_cond_timedwait.c> | ||
3373 | -#else | ||
3374 | -# include <pthread.h> | ||
3375 | -# include <pthreadP.h> | ||
3376 | -# include <internaltypes.h> | ||
3377 | -# include <shlib-compat.h> | ||
3378 | -int | ||
3379 | -__pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, | ||
3380 | - const struct timespec *abstime) | ||
3381 | -{ | ||
3382 | - cond_compat_check_and_clear (cond); | ||
3383 | - return __pthread_cond_timedwait_internal (cond, mutex, abstime); | ||
3384 | -} | ||
3385 | -versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, | ||
3386 | - GLIBC_2_3_2); | ||
3387 | -# undef versioned_symbol | ||
3388 | -# define versioned_symbol(lib, local, symbol, version) | ||
3389 | -# undef __pthread_cond_timedwait | ||
3390 | -# define __pthread_cond_timedwait __pthread_cond_timedwait_internal | ||
3391 | -# include_next <pthread_cond_timedwait.c> | ||
3392 | -#endif | ||
3393 | diff --git a/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c b/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | ||
3394 | index 8f02831..0611f7d 100644 | ||
3395 | --- a/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | ||
3396 | +++ b/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | ||
3397 | @@ -32,9 +32,22 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) | ||
3398 | } | ||
3399 | versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, | ||
3400 | GLIBC_2_3_2); | ||
3401 | +int | ||
3402 | +__pthread_cond_timedwait (cond, mutex, abstime) | ||
3403 | + pthread_cond_t *cond; | ||
3404 | + pthread_mutex_t *mutex; | ||
3405 | + const struct timespec *abstime; | ||
3406 | +{ | ||
3407 | + cond_compat_check_and_clear (cond); | ||
3408 | + return __pthread_cond_timedwait_internal (cond, mutex, abstime); | ||
3409 | +} | ||
3410 | +versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, | ||
3411 | + GLIBC_2_3_2); | ||
3412 | # undef versioned_symbol | ||
3413 | # define versioned_symbol(lib, local, symbol, version) | ||
3414 | # undef __pthread_cond_wait | ||
3415 | # define __pthread_cond_wait __pthread_cond_wait_internal | ||
3416 | +# undef __pthread_cond_timedwait | ||
3417 | +# define __pthread_cond_timedwait __pthread_cond_timedwait_internal | ||
3418 | # include_next <pthread_cond_wait.c> | ||
3419 | #endif | ||
3420 | diff --git a/sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S b/sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S | ||
3421 | deleted file mode 100644 | ||
3422 | index f697e5b..0000000 | ||
3423 | --- a/sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S | ||
3424 | +++ /dev/null | ||
3425 | @@ -1,20 +0,0 @@ | ||
3426 | -/* Copyright (C) 2003-2016 Free Software Foundation, Inc. | ||
3427 | - This file is part of the GNU C Library. | ||
3428 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2003. | ||
3429 | - | ||
3430 | - The GNU C Library is free software; you can redistribute it and/or | ||
3431 | - modify it under the terms of the GNU Lesser General Public | ||
3432 | - License as published by the Free Software Foundation; either | ||
3433 | - version 2.1 of the License, or (at your option) any later version. | ||
3434 | - | ||
3435 | - The GNU C Library is distributed in the hope that it will be useful, | ||
3436 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3437 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
3438 | - Lesser General Public License for more details. | ||
3439 | - | ||
3440 | - You should have received a copy of the GNU Lesser General Public | ||
3441 | - License along with the GNU C Library; if not, see | ||
3442 | - <http://www.gnu.org/licenses/>. */ | ||
3443 | - | ||
3444 | -#define HAVE_CMOV 1 | ||
3445 | -#include "../pthread_cond_timedwait.S" | ||
3446 | diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S | ||
3447 | deleted file mode 100644 | ||
3448 | index 5996688..0000000 | ||
3449 | --- a/sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S | ||
3450 | +++ /dev/null | ||
3451 | @@ -1,241 +0,0 @@ | ||
3452 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
3453 | - This file is part of the GNU C Library. | ||
3454 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
3455 | - | ||
3456 | - The GNU C Library is free software; you can redistribute it and/or | ||
3457 | - modify it under the terms of the GNU Lesser General Public | ||
3458 | - License as published by the Free Software Foundation; either | ||
3459 | - version 2.1 of the License, or (at your option) any later version. | ||
3460 | - | ||
3461 | - The GNU C Library is distributed in the hope that it will be useful, | ||
3462 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3463 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
3464 | - Lesser General Public License for more details. | ||
3465 | - | ||
3466 | - You should have received a copy of the GNU Lesser General Public | ||
3467 | - License along with the GNU C Library; if not, see | ||
3468 | - <http://www.gnu.org/licenses/>. */ | ||
3469 | - | ||
3470 | -#include <sysdep.h> | ||
3471 | -#include <shlib-compat.h> | ||
3472 | -#include <lowlevellock.h> | ||
3473 | -#include <lowlevelcond.h> | ||
3474 | -#include <kernel-features.h> | ||
3475 | -#include <pthread-pi-defines.h> | ||
3476 | -#include <pthread-errnos.h> | ||
3477 | -#include <stap-probe.h> | ||
3478 | - | ||
3479 | - .text | ||
3480 | - | ||
3481 | - /* int pthread_cond_broadcast (pthread_cond_t *cond) */ | ||
3482 | - .globl __pthread_cond_broadcast | ||
3483 | - .type __pthread_cond_broadcast, @function | ||
3484 | - .align 16 | ||
3485 | -__pthread_cond_broadcast: | ||
3486 | - cfi_startproc | ||
3487 | - pushl %ebx | ||
3488 | - cfi_adjust_cfa_offset(4) | ||
3489 | - cfi_rel_offset(%ebx, 0) | ||
3490 | - pushl %esi | ||
3491 | - cfi_adjust_cfa_offset(4) | ||
3492 | - cfi_rel_offset(%esi, 0) | ||
3493 | - pushl %edi | ||
3494 | - cfi_adjust_cfa_offset(4) | ||
3495 | - cfi_rel_offset(%edi, 0) | ||
3496 | - pushl %ebp | ||
3497 | - cfi_adjust_cfa_offset(4) | ||
3498 | - cfi_rel_offset(%ebp, 0) | ||
3499 | - cfi_remember_state | ||
3500 | - | ||
3501 | - movl 20(%esp), %ebx | ||
3502 | - | ||
3503 | - LIBC_PROBE (cond_broadcast, 1, %edx) | ||
3504 | - | ||
3505 | - /* Get internal lock. */ | ||
3506 | - movl $1, %edx | ||
3507 | - xorl %eax, %eax | ||
3508 | - LOCK | ||
3509 | -#if cond_lock == 0 | ||
3510 | - cmpxchgl %edx, (%ebx) | ||
3511 | -#else | ||
3512 | - cmpxchgl %edx, cond_lock(%ebx) | ||
3513 | -#endif | ||
3514 | - jnz 1f | ||
3515 | - | ||
3516 | -2: addl $cond_futex, %ebx | ||
3517 | - movl total_seq+4-cond_futex(%ebx), %eax | ||
3518 | - movl total_seq-cond_futex(%ebx), %ebp | ||
3519 | - cmpl wakeup_seq+4-cond_futex(%ebx), %eax | ||
3520 | - ja 3f | ||
3521 | - jb 4f | ||
3522 | - cmpl wakeup_seq-cond_futex(%ebx), %ebp | ||
3523 | - jna 4f | ||
3524 | - | ||
3525 | - /* Cause all currently waiting threads to recognize they are | ||
3526 | - woken up. */ | ||
3527 | -3: movl %ebp, wakeup_seq-cond_futex(%ebx) | ||
3528 | - movl %eax, wakeup_seq-cond_futex+4(%ebx) | ||
3529 | - movl %ebp, woken_seq-cond_futex(%ebx) | ||
3530 | - movl %eax, woken_seq-cond_futex+4(%ebx) | ||
3531 | - addl %ebp, %ebp | ||
3532 | - addl $1, broadcast_seq-cond_futex(%ebx) | ||
3533 | - movl %ebp, (%ebx) | ||
3534 | - | ||
3535 | - /* Get the address of the mutex used. */ | ||
3536 | - movl dep_mutex-cond_futex(%ebx), %edi | ||
3537 | - | ||
3538 | - /* Unlock. */ | ||
3539 | - LOCK | ||
3540 | - subl $1, cond_lock-cond_futex(%ebx) | ||
3541 | - jne 7f | ||
3542 | - | ||
3543 | - /* Don't use requeue for pshared condvars. */ | ||
3544 | -8: cmpl $-1, %edi | ||
3545 | - je 9f | ||
3546 | - | ||
3547 | - /* Do not use requeue for pshared condvars. */ | ||
3548 | - testl $PS_BIT, MUTEX_KIND(%edi) | ||
3549 | - jne 9f | ||
3550 | - | ||
3551 | - /* Requeue to a non-robust PI mutex if the PI bit is set and | ||
3552 | - the robust bit is not set. */ | ||
3553 | - movl MUTEX_KIND(%edi), %eax | ||
3554 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
3555 | - cmpl $PI_BIT, %eax | ||
3556 | - je 81f | ||
3557 | - | ||
3558 | - /* Wake up all threads. */ | ||
3559 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
3560 | - movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx | ||
3561 | -#else | ||
3562 | - movl %gs:PRIVATE_FUTEX, %ecx | ||
3563 | - orl $FUTEX_CMP_REQUEUE, %ecx | ||
3564 | -#endif | ||
3565 | - movl $SYS_futex, %eax | ||
3566 | - movl $0x7fffffff, %esi | ||
3567 | - movl $1, %edx | ||
3568 | - /* Get the address of the futex involved. */ | ||
3569 | -# if MUTEX_FUTEX != 0 | ||
3570 | - addl $MUTEX_FUTEX, %edi | ||
3571 | -# endif | ||
3572 | -/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for sysenter. | ||
3573 | - ENTER_KERNEL */ | ||
3574 | - int $0x80 | ||
3575 | - | ||
3576 | - /* For any kind of error, which mainly is EAGAIN, we try again | ||
3577 | - with WAKE. The general test also covers running on old | ||
3578 | - kernels. */ | ||
3579 | - cmpl $0xfffff001, %eax | ||
3580 | - jae 9f | ||
3581 | - | ||
3582 | -6: xorl %eax, %eax | ||
3583 | - popl %ebp | ||
3584 | - cfi_adjust_cfa_offset(-4) | ||
3585 | - cfi_restore(%ebp) | ||
3586 | - popl %edi | ||
3587 | - cfi_adjust_cfa_offset(-4) | ||
3588 | - cfi_restore(%edi) | ||
3589 | - popl %esi | ||
3590 | - cfi_adjust_cfa_offset(-4) | ||
3591 | - cfi_restore(%esi) | ||
3592 | - popl %ebx | ||
3593 | - cfi_adjust_cfa_offset(-4) | ||
3594 | - cfi_restore(%ebx) | ||
3595 | - ret | ||
3596 | - | ||
3597 | - cfi_restore_state | ||
3598 | - | ||
3599 | -81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx | ||
3600 | - movl $SYS_futex, %eax | ||
3601 | - movl $0x7fffffff, %esi | ||
3602 | - movl $1, %edx | ||
3603 | - /* Get the address of the futex involved. */ | ||
3604 | -# if MUTEX_FUTEX != 0 | ||
3605 | - addl $MUTEX_FUTEX, %edi | ||
3606 | -# endif | ||
3607 | - int $0x80 | ||
3608 | - | ||
3609 | - /* For any kind of error, which mainly is EAGAIN, we try again | ||
3610 | - with WAKE. The general test also covers running on old | ||
3611 | - kernels. */ | ||
3612 | - cmpl $0xfffff001, %eax | ||
3613 | - jb 6b | ||
3614 | - jmp 9f | ||
3615 | - | ||
3616 | - /* Initial locking failed. */ | ||
3617 | -1: | ||
3618 | -#if cond_lock == 0 | ||
3619 | - movl %ebx, %edx | ||
3620 | -#else | ||
3621 | - leal cond_lock(%ebx), %edx | ||
3622 | -#endif | ||
3623 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
3624 | - xorl %ecx, %ecx | ||
3625 | -#endif | ||
3626 | - cmpl $-1, dep_mutex(%ebx) | ||
3627 | - setne %cl | ||
3628 | - subl $1, %ecx | ||
3629 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
3630 | -#if LLL_PRIVATE != 0 | ||
3631 | - addl $LLL_PRIVATE, %ecx | ||
3632 | -#endif | ||
3633 | - call __lll_lock_wait | ||
3634 | - jmp 2b | ||
3635 | - | ||
3636 | - .align 16 | ||
3637 | - /* Unlock. */ | ||
3638 | -4: LOCK | ||
3639 | - subl $1, cond_lock-cond_futex(%ebx) | ||
3640 | - je 6b | ||
3641 | - | ||
3642 | - /* Unlock in loop requires wakeup. */ | ||
3643 | -5: leal cond_lock-cond_futex(%ebx), %eax | ||
3644 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
3645 | - xorl %ecx, %ecx | ||
3646 | -#endif | ||
3647 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
3648 | - setne %cl | ||
3649 | - subl $1, %ecx | ||
3650 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
3651 | -#if LLL_PRIVATE != 0 | ||
3652 | - addl $LLL_PRIVATE, %ecx | ||
3653 | -#endif | ||
3654 | - call __lll_unlock_wake | ||
3655 | - jmp 6b | ||
3656 | - | ||
3657 | - /* Unlock in loop requires wakeup. */ | ||
3658 | -7: leal cond_lock-cond_futex(%ebx), %eax | ||
3659 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
3660 | - xorl %ecx, %ecx | ||
3661 | -#endif | ||
3662 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
3663 | - setne %cl | ||
3664 | - subl $1, %ecx | ||
3665 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
3666 | -#if LLL_PRIVATE != 0 | ||
3667 | - addl $LLL_PRIVATE, %ecx | ||
3668 | -#endif | ||
3669 | - call __lll_unlock_wake | ||
3670 | - jmp 8b | ||
3671 | - | ||
3672 | -9: /* The futex requeue functionality is not available. */ | ||
3673 | - movl $0x7fffffff, %edx | ||
3674 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
3675 | - xorl %ecx, %ecx | ||
3676 | -#endif | ||
3677 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
3678 | - sete %cl | ||
3679 | - subl $1, %ecx | ||
3680 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
3681 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
3682 | -#else | ||
3683 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
3684 | -#endif | ||
3685 | - addl $FUTEX_WAKE, %ecx | ||
3686 | - movl $SYS_futex, %eax | ||
3687 | - ENTER_KERNEL | ||
3688 | - jmp 6b | ||
3689 | - cfi_endproc | ||
3690 | - .size __pthread_cond_broadcast, .-__pthread_cond_broadcast | ||
3691 | -versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast, | ||
3692 | - GLIBC_2_3_2) | ||
3693 | diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S | ||
3694 | deleted file mode 100644 | ||
3695 | index 0038775..0000000 | ||
3696 | --- a/sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S | ||
3697 | +++ /dev/null | ||
3698 | @@ -1,216 +0,0 @@ | ||
3699 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
3700 | - This file is part of the GNU C Library. | ||
3701 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
3702 | - | ||
3703 | - The GNU C Library is free software; you can redistribute it and/or | ||
3704 | - modify it under the terms of the GNU Lesser General Public | ||
3705 | - License as published by the Free Software Foundation; either | ||
3706 | - version 2.1 of the License, or (at your option) any later version. | ||
3707 | - | ||
3708 | - The GNU C Library is distributed in the hope that it will be useful, | ||
3709 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3710 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
3711 | - Lesser General Public License for more details. | ||
3712 | - | ||
3713 | - You should have received a copy of the GNU Lesser General Public | ||
3714 | - License along with the GNU C Library; if not, see | ||
3715 | - <http://www.gnu.org/licenses/>. */ | ||
3716 | - | ||
3717 | -#include <sysdep.h> | ||
3718 | -#include <shlib-compat.h> | ||
3719 | -#include <lowlevellock.h> | ||
3720 | -#include <lowlevelcond.h> | ||
3721 | -#include <kernel-features.h> | ||
3722 | -#include <pthread-pi-defines.h> | ||
3723 | -#include <pthread-errnos.h> | ||
3724 | -#include <stap-probe.h> | ||
3725 | - | ||
3726 | - .text | ||
3727 | - | ||
3728 | - /* int pthread_cond_signal (pthread_cond_t *cond) */ | ||
3729 | - .globl __pthread_cond_signal | ||
3730 | - .type __pthread_cond_signal, @function | ||
3731 | - .align 16 | ||
3732 | -__pthread_cond_signal: | ||
3733 | - | ||
3734 | - cfi_startproc | ||
3735 | - pushl %ebx | ||
3736 | - cfi_adjust_cfa_offset(4) | ||
3737 | - cfi_rel_offset(%ebx, 0) | ||
3738 | - pushl %edi | ||
3739 | - cfi_adjust_cfa_offset(4) | ||
3740 | - cfi_rel_offset(%edi, 0) | ||
3741 | - cfi_remember_state | ||
3742 | - | ||
3743 | - movl 12(%esp), %edi | ||
3744 | - | ||
3745 | - LIBC_PROBE (cond_signal, 1, %edi) | ||
3746 | - | ||
3747 | - /* Get internal lock. */ | ||
3748 | - movl $1, %edx | ||
3749 | - xorl %eax, %eax | ||
3750 | - LOCK | ||
3751 | -#if cond_lock == 0 | ||
3752 | - cmpxchgl %edx, (%edi) | ||
3753 | -#else | ||
3754 | - cmpxchgl %edx, cond_lock(%edi) | ||
3755 | -#endif | ||
3756 | - jnz 1f | ||
3757 | - | ||
3758 | -2: leal cond_futex(%edi), %ebx | ||
3759 | - movl total_seq+4(%edi), %eax | ||
3760 | - movl total_seq(%edi), %ecx | ||
3761 | - cmpl wakeup_seq+4(%edi), %eax | ||
3762 | -#if cond_lock != 0 | ||
3763 | - /* Must use leal to preserve the flags. */ | ||
3764 | - leal cond_lock(%edi), %edi | ||
3765 | -#endif | ||
3766 | - ja 3f | ||
3767 | - jb 4f | ||
3768 | - cmpl wakeup_seq-cond_futex(%ebx), %ecx | ||
3769 | - jbe 4f | ||
3770 | - | ||
3771 | - /* Bump the wakeup number. */ | ||
3772 | -3: addl $1, wakeup_seq-cond_futex(%ebx) | ||
3773 | - adcl $0, wakeup_seq-cond_futex+4(%ebx) | ||
3774 | - addl $1, (%ebx) | ||
3775 | - | ||
3776 | - /* Wake up one thread. */ | ||
3777 | - pushl %esi | ||
3778 | - cfi_adjust_cfa_offset(4) | ||
3779 | - cfi_rel_offset(%esi, 0) | ||
3780 | - pushl %ebp | ||
3781 | - cfi_adjust_cfa_offset(4) | ||
3782 | - cfi_rel_offset(%ebp, 0) | ||
3783 | - | ||
3784 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
3785 | - xorl %ecx, %ecx | ||
3786 | -#endif | ||
3787 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
3788 | - sete %cl | ||
3789 | - je 8f | ||
3790 | - | ||
3791 | - movl dep_mutex-cond_futex(%ebx), %edx | ||
3792 | - /* Requeue to a non-robust PI mutex if the PI bit is set and | ||
3793 | - the robust bit is not set. */ | ||
3794 | - movl MUTEX_KIND(%edx), %eax | ||
3795 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
3796 | - cmpl $PI_BIT, %eax | ||
3797 | - je 9f | ||
3798 | - | ||
3799 | -8: subl $1, %ecx | ||
3800 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
3801 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
3802 | -#else | ||
3803 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
3804 | -#endif | ||
3805 | - addl $FUTEX_WAKE_OP, %ecx | ||
3806 | - movl $SYS_futex, %eax | ||
3807 | - movl $1, %edx | ||
3808 | - movl $1, %esi | ||
3809 | - movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %ebp | ||
3810 | - /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for | ||
3811 | - sysenter. | ||
3812 | - ENTER_KERNEL */ | ||
3813 | - int $0x80 | ||
3814 | - popl %ebp | ||
3815 | - cfi_adjust_cfa_offset(-4) | ||
3816 | - cfi_restore(%ebp) | ||
3817 | - popl %esi | ||
3818 | - cfi_adjust_cfa_offset(-4) | ||
3819 | - cfi_restore(%esi) | ||
3820 | - | ||
3821 | - /* For any kind of error, we try again with WAKE. | ||
3822 | - The general test also covers running on old kernels. */ | ||
3823 | - cmpl $-4095, %eax | ||
3824 | - jae 7f | ||
3825 | - | ||
3826 | -6: xorl %eax, %eax | ||
3827 | - popl %edi | ||
3828 | - cfi_adjust_cfa_offset(-4) | ||
3829 | - cfi_restore(%edi) | ||
3830 | - popl %ebx | ||
3831 | - cfi_adjust_cfa_offset(-4) | ||
3832 | - cfi_restore(%ebx) | ||
3833 | - ret | ||
3834 | - | ||
3835 | - cfi_restore_state | ||
3836 | - | ||
3837 | -9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx | ||
3838 | - movl $SYS_futex, %eax | ||
3839 | - movl $1, %edx | ||
3840 | - xorl %esi, %esi | ||
3841 | - movl dep_mutex-cond_futex(%ebx), %edi | ||
3842 | - movl (%ebx), %ebp | ||
3843 | - /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for | ||
3844 | - sysenter. | ||
3845 | - ENTER_KERNEL */ | ||
3846 | - int $0x80 | ||
3847 | - popl %ebp | ||
3848 | - popl %esi | ||
3849 | - | ||
3850 | - leal -cond_futex(%ebx), %edi | ||
3851 | - | ||
3852 | - /* For any kind of error, we try again with WAKE. | ||
3853 | - The general test also covers running on old kernels. */ | ||
3854 | - cmpl $-4095, %eax | ||
3855 | - jb 4f | ||
3856 | - | ||
3857 | -7: | ||
3858 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
3859 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
3860 | -#else | ||
3861 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
3862 | -#endif | ||
3863 | - orl $FUTEX_WAKE, %ecx | ||
3864 | - | ||
3865 | - movl $SYS_futex, %eax | ||
3866 | - /* %edx should be 1 already from $FUTEX_WAKE_OP syscall. | ||
3867 | - movl $1, %edx */ | ||
3868 | - ENTER_KERNEL | ||
3869 | - | ||
3870 | - /* Unlock. Note that at this point %edi always points to | ||
3871 | - cond_lock. */ | ||
3872 | -4: LOCK | ||
3873 | - subl $1, (%edi) | ||
3874 | - je 6b | ||
3875 | - | ||
3876 | - /* Unlock in loop requires wakeup. */ | ||
3877 | -5: movl %edi, %eax | ||
3878 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
3879 | - xorl %ecx, %ecx | ||
3880 | -#endif | ||
3881 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
3882 | - setne %cl | ||
3883 | - subl $1, %ecx | ||
3884 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
3885 | -#if LLL_PRIVATE != 0 | ||
3886 | - addl $LLL_PRIVATE, %ecx | ||
3887 | -#endif | ||
3888 | - call __lll_unlock_wake | ||
3889 | - jmp 6b | ||
3890 | - | ||
3891 | - /* Initial locking failed. */ | ||
3892 | -1: | ||
3893 | -#if cond_lock == 0 | ||
3894 | - movl %edi, %edx | ||
3895 | -#else | ||
3896 | - leal cond_lock(%edi), %edx | ||
3897 | -#endif | ||
3898 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
3899 | - xorl %ecx, %ecx | ||
3900 | -#endif | ||
3901 | - cmpl $-1, dep_mutex(%edi) | ||
3902 | - setne %cl | ||
3903 | - subl $1, %ecx | ||
3904 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
3905 | -#if LLL_PRIVATE != 0 | ||
3906 | - addl $LLL_PRIVATE, %ecx | ||
3907 | -#endif | ||
3908 | - call __lll_lock_wait | ||
3909 | - jmp 2b | ||
3910 | - | ||
3911 | - cfi_endproc | ||
3912 | - .size __pthread_cond_signal, .-__pthread_cond_signal | ||
3913 | -versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, | ||
3914 | - GLIBC_2_3_2) | ||
3915 | diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S | ||
3916 | deleted file mode 100644 | ||
3917 | index 6256376..0000000 | ||
3918 | --- a/sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S | ||
3919 | +++ /dev/null | ||
3920 | @@ -1,974 +0,0 @@ | ||
3921 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
3922 | - This file is part of the GNU C Library. | ||
3923 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
3924 | - | ||
3925 | - The GNU C Library is free software; you can redistribute it and/or | ||
3926 | - modify it under the terms of the GNU Lesser General Public | ||
3927 | - License as published by the Free Software Foundation; either | ||
3928 | - version 2.1 of the License, or (at your option) any later version. | ||
3929 | - | ||
3930 | - The GNU C Library is distributed in the hope that it will be useful, | ||
3931 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
3932 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
3933 | - Lesser General Public License for more details. | ||
3934 | - | ||
3935 | - You should have received a copy of the GNU Lesser General Public | ||
3936 | - License along with the GNU C Library; if not, see | ||
3937 | - <http://www.gnu.org/licenses/>. */ | ||
3938 | - | ||
3939 | -#include <sysdep.h> | ||
3940 | -#include <shlib-compat.h> | ||
3941 | -#include <lowlevellock.h> | ||
3942 | -#include <lowlevelcond.h> | ||
3943 | -#include <pthread-errnos.h> | ||
3944 | -#include <pthread-pi-defines.h> | ||
3945 | -#include <kernel-features.h> | ||
3946 | -#include <stap-probe.h> | ||
3947 | - | ||
3948 | - .text | ||
3949 | - | ||
3950 | -/* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, | ||
3951 | - const struct timespec *abstime) */ | ||
3952 | - .globl __pthread_cond_timedwait | ||
3953 | - .type __pthread_cond_timedwait, @function | ||
3954 | - .align 16 | ||
3955 | -__pthread_cond_timedwait: | ||
3956 | -.LSTARTCODE: | ||
3957 | - cfi_startproc | ||
3958 | -#ifdef SHARED | ||
3959 | - cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, | ||
3960 | - DW.ref.__gcc_personality_v0) | ||
3961 | - cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) | ||
3962 | -#else | ||
3963 | - cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) | ||
3964 | - cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) | ||
3965 | -#endif | ||
3966 | - | ||
3967 | - pushl %ebp | ||
3968 | - cfi_adjust_cfa_offset(4) | ||
3969 | - cfi_rel_offset(%ebp, 0) | ||
3970 | - pushl %edi | ||
3971 | - cfi_adjust_cfa_offset(4) | ||
3972 | - cfi_rel_offset(%edi, 0) | ||
3973 | - pushl %esi | ||
3974 | - cfi_adjust_cfa_offset(4) | ||
3975 | - cfi_rel_offset(%esi, 0) | ||
3976 | - pushl %ebx | ||
3977 | - cfi_adjust_cfa_offset(4) | ||
3978 | - cfi_rel_offset(%ebx, 0) | ||
3979 | - | ||
3980 | - movl 20(%esp), %ebx | ||
3981 | - movl 28(%esp), %ebp | ||
3982 | - | ||
3983 | - LIBC_PROBE (cond_timedwait, 3, %ebx, 24(%esp), %ebp) | ||
3984 | - | ||
3985 | - cmpl $1000000000, 4(%ebp) | ||
3986 | - movl $EINVAL, %eax | ||
3987 | - jae 18f | ||
3988 | - | ||
3989 | - /* Stack frame: | ||
3990 | - | ||
3991 | - esp + 32 | ||
3992 | - +--------------------------+ | ||
3993 | - esp + 24 | timeout value | | ||
3994 | - +--------------------------+ | ||
3995 | - esp + 20 | futex pointer | | ||
3996 | - +--------------------------+ | ||
3997 | - esp + 16 | pi-requeued flag | | ||
3998 | - +--------------------------+ | ||
3999 | - esp + 12 | old broadcast_seq value | | ||
4000 | - +--------------------------+ | ||
4001 | - esp + 4 | old wake_seq value | | ||
4002 | - +--------------------------+ | ||
4003 | - esp + 0 | old cancellation mode | | ||
4004 | - +--------------------------+ | ||
4005 | - */ | ||
4006 | - | ||
4007 | -#ifndef __ASSUME_FUTEX_CLOCK_REALTIME | ||
4008 | -# ifdef PIC | ||
4009 | - LOAD_PIC_REG (cx) | ||
4010 | - cmpl $0, __have_futex_clock_realtime@GOTOFF(%ecx) | ||
4011 | -# else | ||
4012 | - cmpl $0, __have_futex_clock_realtime | ||
4013 | -# endif | ||
4014 | - je .Lreltmo | ||
4015 | -#endif | ||
4016 | - | ||
4017 | - /* Get internal lock. */ | ||
4018 | - movl $1, %edx | ||
4019 | - xorl %eax, %eax | ||
4020 | - LOCK | ||
4021 | -#if cond_lock == 0 | ||
4022 | - cmpxchgl %edx, (%ebx) | ||
4023 | -#else | ||
4024 | - cmpxchgl %edx, cond_lock(%ebx) | ||
4025 | -#endif | ||
4026 | - jnz 1f | ||
4027 | - | ||
4028 | - /* Store the reference to the mutex. If there is already a | ||
4029 | - different value in there this is a bad user bug. */ | ||
4030 | -2: cmpl $-1, dep_mutex(%ebx) | ||
4031 | - movl 24(%esp), %eax | ||
4032 | - je 17f | ||
4033 | - movl %eax, dep_mutex(%ebx) | ||
4034 | - | ||
4035 | - /* Unlock the mutex. */ | ||
4036 | -17: xorl %edx, %edx | ||
4037 | - call __pthread_mutex_unlock_usercnt | ||
4038 | - | ||
4039 | - testl %eax, %eax | ||
4040 | - jne 16f | ||
4041 | - | ||
4042 | - addl $1, total_seq(%ebx) | ||
4043 | - adcl $0, total_seq+4(%ebx) | ||
4044 | - addl $1, cond_futex(%ebx) | ||
4045 | - addl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
4046 | - | ||
4047 | -#ifdef __ASSUME_FUTEX_CLOCK_REALTIME | ||
4048 | -# define FRAME_SIZE 24 | ||
4049 | -#else | ||
4050 | -# define FRAME_SIZE 32 | ||
4051 | -#endif | ||
4052 | - subl $FRAME_SIZE, %esp | ||
4053 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
4054 | - cfi_remember_state | ||
4055 | - | ||
4056 | - /* Get and store current wakeup_seq value. */ | ||
4057 | - movl wakeup_seq(%ebx), %edi | ||
4058 | - movl wakeup_seq+4(%ebx), %edx | ||
4059 | - movl broadcast_seq(%ebx), %eax | ||
4060 | - movl %edi, 4(%esp) | ||
4061 | - movl %edx, 8(%esp) | ||
4062 | - movl %eax, 12(%esp) | ||
4063 | - | ||
4064 | - /* Reset the pi-requeued flag. */ | ||
4065 | - movl $0, 16(%esp) | ||
4066 | - | ||
4067 | - cmpl $0, (%ebp) | ||
4068 | - movl $-ETIMEDOUT, %esi | ||
4069 | - js 6f | ||
4070 | - | ||
4071 | -8: movl cond_futex(%ebx), %edi | ||
4072 | - movl %edi, 20(%esp) | ||
4073 | - | ||
4074 | - /* Unlock. */ | ||
4075 | - LOCK | ||
4076 | -#if cond_lock == 0 | ||
4077 | - subl $1, (%ebx) | ||
4078 | -#else | ||
4079 | - subl $1, cond_lock(%ebx) | ||
4080 | -#endif | ||
4081 | - jne 3f | ||
4082 | - | ||
4083 | -.LcleanupSTART: | ||
4084 | -4: call __pthread_enable_asynccancel | ||
4085 | - movl %eax, (%esp) | ||
4086 | - | ||
4087 | - leal (%ebp), %esi | ||
4088 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
4089 | - xorl %ecx, %ecx | ||
4090 | -#endif | ||
4091 | - cmpl $-1, dep_mutex(%ebx) | ||
4092 | - sete %cl | ||
4093 | - je 40f | ||
4094 | - | ||
4095 | - movl dep_mutex(%ebx), %edi | ||
4096 | - /* Requeue to a non-robust PI mutex if the PI bit is set and | ||
4097 | - the robust bit is not set. */ | ||
4098 | - movl MUTEX_KIND(%edi), %eax | ||
4099 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
4100 | - cmpl $PI_BIT, %eax | ||
4101 | - jne 40f | ||
4102 | - | ||
4103 | - movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx | ||
4104 | - /* The following only works like this because we only support | ||
4105 | - two clocks, represented using a single bit. */ | ||
4106 | - testl $1, cond_nwaiters(%ebx) | ||
4107 | - /* XXX Need to implement using sete instead of a jump. */ | ||
4108 | - jne 42f | ||
4109 | - orl $FUTEX_CLOCK_REALTIME, %ecx | ||
4110 | - | ||
4111 | -42: movl 20(%esp), %edx | ||
4112 | - addl $cond_futex, %ebx | ||
4113 | -.Ladd_cond_futex_pi: | ||
4114 | - movl $SYS_futex, %eax | ||
4115 | - ENTER_KERNEL | ||
4116 | - subl $cond_futex, %ebx | ||
4117 | -.Lsub_cond_futex_pi: | ||
4118 | - movl %eax, %esi | ||
4119 | - /* Set the pi-requeued flag only if the kernel has returned 0. The | ||
4120 | - kernel does not hold the mutex on ETIMEDOUT or any other error. */ | ||
4121 | - cmpl $0, %eax | ||
4122 | - sete 16(%esp) | ||
4123 | - je 41f | ||
4124 | - | ||
4125 | - /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns | ||
4126 | - successfully, it has already locked the mutex for us and the | ||
4127 | - pi_flag (16(%esp)) is set to denote that fact. However, if another | ||
4128 | - thread changed the futex value before we entered the wait, the | ||
4129 | - syscall may return an EAGAIN and the mutex is not locked. We go | ||
4130 | - ahead with a success anyway since later we look at the pi_flag to | ||
4131 | - decide if we got the mutex or not. The sequence numbers then make | ||
4132 | - sure that only one of the threads actually wake up. We retry using | ||
4133 | - normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal | ||
4134 | - and PI futexes don't mix. | ||
4135 | - | ||
4136 | - Note that we don't check for EAGAIN specifically; we assume that the | ||
4137 | - only other error the futex function could return is EAGAIN (barring | ||
4138 | - the ETIMEOUT of course, for the timeout case in futex) since | ||
4139 | - anything else would mean an error in our function. It is too | ||
4140 | - expensive to do that check for every call (which is quite common in | ||
4141 | - case of a large number of threads), so it has been skipped. */ | ||
4142 | - cmpl $-ENOSYS, %eax | ||
4143 | - jne 41f | ||
4144 | - xorl %ecx, %ecx | ||
4145 | - | ||
4146 | -40: subl $1, %ecx | ||
4147 | - movl $0, 16(%esp) | ||
4148 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
4149 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
4150 | -#else | ||
4151 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
4152 | -#endif | ||
4153 | - addl $FUTEX_WAIT_BITSET, %ecx | ||
4154 | - /* The following only works like this because we only support | ||
4155 | - two clocks, represented using a single bit. */ | ||
4156 | - testl $1, cond_nwaiters(%ebx) | ||
4157 | - jne 30f | ||
4158 | - orl $FUTEX_CLOCK_REALTIME, %ecx | ||
4159 | -30: | ||
4160 | - movl 20(%esp), %edx | ||
4161 | - movl $0xffffffff, %ebp | ||
4162 | - addl $cond_futex, %ebx | ||
4163 | -.Ladd_cond_futex: | ||
4164 | - movl $SYS_futex, %eax | ||
4165 | - ENTER_KERNEL | ||
4166 | - subl $cond_futex, %ebx | ||
4167 | -.Lsub_cond_futex: | ||
4168 | - movl 28+FRAME_SIZE(%esp), %ebp | ||
4169 | - movl %eax, %esi | ||
4170 | - | ||
4171 | -41: movl (%esp), %eax | ||
4172 | - call __pthread_disable_asynccancel | ||
4173 | -.LcleanupEND: | ||
4174 | - | ||
4175 | - /* Lock. */ | ||
4176 | - movl $1, %edx | ||
4177 | - xorl %eax, %eax | ||
4178 | - LOCK | ||
4179 | -#if cond_lock == 0 | ||
4180 | - cmpxchgl %edx, (%ebx) | ||
4181 | -#else | ||
4182 | - cmpxchgl %edx, cond_lock(%ebx) | ||
4183 | -#endif | ||
4184 | - jnz 5f | ||
4185 | - | ||
4186 | -6: movl broadcast_seq(%ebx), %eax | ||
4187 | - cmpl 12(%esp), %eax | ||
4188 | - jne 23f | ||
4189 | - | ||
4190 | - movl woken_seq(%ebx), %eax | ||
4191 | - movl woken_seq+4(%ebx), %ecx | ||
4192 | - | ||
4193 | - movl wakeup_seq(%ebx), %edi | ||
4194 | - movl wakeup_seq+4(%ebx), %edx | ||
4195 | - | ||
4196 | - cmpl 8(%esp), %edx | ||
4197 | - jne 7f | ||
4198 | - cmpl 4(%esp), %edi | ||
4199 | - je 15f | ||
4200 | - | ||
4201 | -7: cmpl %ecx, %edx | ||
4202 | - jne 9f | ||
4203 | - cmp %eax, %edi | ||
4204 | - jne 9f | ||
4205 | - | ||
4206 | -15: cmpl $-ETIMEDOUT, %esi | ||
4207 | - je 28f | ||
4208 | - | ||
4209 | - /* We need to go back to futex_wait. If we're using requeue_pi, then | ||
4210 | - release the mutex we had acquired and go back. */ | ||
4211 | - movl 16(%esp), %edx | ||
4212 | - test %edx, %edx | ||
4213 | - jz 8b | ||
4214 | - | ||
4215 | - /* Adjust the mutex values first and then unlock it. The unlock | ||
4216 | - should always succeed or else the kernel did not lock the mutex | ||
4217 | - correctly. */ | ||
4218 | - movl dep_mutex(%ebx), %eax | ||
4219 | - call __pthread_mutex_cond_lock_adjust | ||
4220 | - movl dep_mutex(%ebx), %eax | ||
4221 | - xorl %edx, %edx | ||
4222 | - call __pthread_mutex_unlock_usercnt | ||
4223 | - jmp 8b | ||
4224 | - | ||
4225 | -28: addl $1, wakeup_seq(%ebx) | ||
4226 | - adcl $0, wakeup_seq+4(%ebx) | ||
4227 | - addl $1, cond_futex(%ebx) | ||
4228 | - movl $ETIMEDOUT, %esi | ||
4229 | - jmp 14f | ||
4230 | - | ||
4231 | -23: xorl %esi, %esi | ||
4232 | - jmp 24f | ||
4233 | - | ||
4234 | -9: xorl %esi, %esi | ||
4235 | -14: addl $1, woken_seq(%ebx) | ||
4236 | - adcl $0, woken_seq+4(%ebx) | ||
4237 | - | ||
4238 | -24: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
4239 | - | ||
4240 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
4241 | - movl total_seq(%ebx), %eax | ||
4242 | - andl total_seq+4(%ebx), %eax | ||
4243 | - cmpl $0xffffffff, %eax | ||
4244 | - jne 25f | ||
4245 | - movl cond_nwaiters(%ebx), %eax | ||
4246 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
4247 | - jne 25f | ||
4248 | - | ||
4249 | - addl $cond_nwaiters, %ebx | ||
4250 | - movl $SYS_futex, %eax | ||
4251 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
4252 | - xorl %ecx, %ecx | ||
4253 | -#endif | ||
4254 | - cmpl $-1, dep_mutex-cond_nwaiters(%ebx) | ||
4255 | - sete %cl | ||
4256 | - subl $1, %ecx | ||
4257 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
4258 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
4259 | -#else | ||
4260 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
4261 | -#endif | ||
4262 | - addl $FUTEX_WAKE, %ecx | ||
4263 | - movl $1, %edx | ||
4264 | - ENTER_KERNEL | ||
4265 | - subl $cond_nwaiters, %ebx | ||
4266 | - | ||
4267 | -25: LOCK | ||
4268 | -#if cond_lock == 0 | ||
4269 | - subl $1, (%ebx) | ||
4270 | -#else | ||
4271 | - subl $1, cond_lock(%ebx) | ||
4272 | -#endif | ||
4273 | - jne 10f | ||
4274 | - | ||
4275 | -11: movl 24+FRAME_SIZE(%esp), %eax | ||
4276 | - /* With requeue_pi, the mutex lock is held in the kernel. */ | ||
4277 | - movl 16(%esp), %ecx | ||
4278 | - testl %ecx, %ecx | ||
4279 | - jnz 27f | ||
4280 | - | ||
4281 | - call __pthread_mutex_cond_lock | ||
4282 | -26: addl $FRAME_SIZE, %esp | ||
4283 | - cfi_adjust_cfa_offset(-FRAME_SIZE) | ||
4284 | - | ||
4285 | - /* We return the result of the mutex_lock operation if it failed. */ | ||
4286 | - testl %eax, %eax | ||
4287 | -#ifdef HAVE_CMOV | ||
4288 | - cmovel %esi, %eax | ||
4289 | -#else | ||
4290 | - jne 22f | ||
4291 | - movl %esi, %eax | ||
4292 | -22: | ||
4293 | -#endif | ||
4294 | - | ||
4295 | -18: popl %ebx | ||
4296 | - cfi_adjust_cfa_offset(-4) | ||
4297 | - cfi_restore(%ebx) | ||
4298 | - popl %esi | ||
4299 | - cfi_adjust_cfa_offset(-4) | ||
4300 | - cfi_restore(%esi) | ||
4301 | - popl %edi | ||
4302 | - cfi_adjust_cfa_offset(-4) | ||
4303 | - cfi_restore(%edi) | ||
4304 | - popl %ebp | ||
4305 | - cfi_adjust_cfa_offset(-4) | ||
4306 | - cfi_restore(%ebp) | ||
4307 | - | ||
4308 | - ret | ||
4309 | - | ||
4310 | - cfi_restore_state | ||
4311 | - | ||
4312 | -27: call __pthread_mutex_cond_lock_adjust | ||
4313 | - xorl %eax, %eax | ||
4314 | - jmp 26b | ||
4315 | - | ||
4316 | - cfi_adjust_cfa_offset(-FRAME_SIZE); | ||
4317 | - /* Initial locking failed. */ | ||
4318 | -1: | ||
4319 | -#if cond_lock == 0 | ||
4320 | - movl %ebx, %edx | ||
4321 | -#else | ||
4322 | - leal cond_lock(%ebx), %edx | ||
4323 | -#endif | ||
4324 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4325 | - xorl %ecx, %ecx | ||
4326 | -#endif | ||
4327 | - cmpl $-1, dep_mutex(%ebx) | ||
4328 | - setne %cl | ||
4329 | - subl $1, %ecx | ||
4330 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4331 | -#if LLL_PRIVATE != 0 | ||
4332 | - addl $LLL_PRIVATE, %ecx | ||
4333 | -#endif | ||
4334 | - call __lll_lock_wait | ||
4335 | - jmp 2b | ||
4336 | - | ||
4337 | - /* The initial unlocking of the mutex failed. */ | ||
4338 | -16: | ||
4339 | - LOCK | ||
4340 | -#if cond_lock == 0 | ||
4341 | - subl $1, (%ebx) | ||
4342 | -#else | ||
4343 | - subl $1, cond_lock(%ebx) | ||
4344 | -#endif | ||
4345 | - jne 18b | ||
4346 | - | ||
4347 | - movl %eax, %esi | ||
4348 | -#if cond_lock == 0 | ||
4349 | - movl %ebx, %eax | ||
4350 | -#else | ||
4351 | - leal cond_lock(%ebx), %eax | ||
4352 | -#endif | ||
4353 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4354 | - xorl %ecx, %ecx | ||
4355 | -#endif | ||
4356 | - cmpl $-1, dep_mutex(%ebx) | ||
4357 | - setne %cl | ||
4358 | - subl $1, %ecx | ||
4359 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4360 | -#if LLL_PRIVATE != 0 | ||
4361 | - addl $LLL_PRIVATE, %ecx | ||
4362 | -#endif | ||
4363 | - call __lll_unlock_wake | ||
4364 | - | ||
4365 | - movl %esi, %eax | ||
4366 | - jmp 18b | ||
4367 | - | ||
4368 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
4369 | - | ||
4370 | - /* Unlock in loop requires wakeup. */ | ||
4371 | -3: | ||
4372 | -#if cond_lock == 0 | ||
4373 | - movl %ebx, %eax | ||
4374 | -#else | ||
4375 | - leal cond_lock(%ebx), %eax | ||
4376 | -#endif | ||
4377 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4378 | - xorl %ecx, %ecx | ||
4379 | -#endif | ||
4380 | - cmpl $-1, dep_mutex(%ebx) | ||
4381 | - setne %cl | ||
4382 | - subl $1, %ecx | ||
4383 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4384 | -#if LLL_PRIVATE != 0 | ||
4385 | - addl $LLL_PRIVATE, %ecx | ||
4386 | -#endif | ||
4387 | - call __lll_unlock_wake | ||
4388 | - jmp 4b | ||
4389 | - | ||
4390 | - /* Locking in loop failed. */ | ||
4391 | -5: | ||
4392 | -#if cond_lock == 0 | ||
4393 | - movl %ebx, %edx | ||
4394 | -#else | ||
4395 | - leal cond_lock(%ebx), %edx | ||
4396 | -#endif | ||
4397 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4398 | - xorl %ecx, %ecx | ||
4399 | -#endif | ||
4400 | - cmpl $-1, dep_mutex(%ebx) | ||
4401 | - setne %cl | ||
4402 | - subl $1, %ecx | ||
4403 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4404 | -#if LLL_PRIVATE != 0 | ||
4405 | - addl $LLL_PRIVATE, %ecx | ||
4406 | -#endif | ||
4407 | - call __lll_lock_wait | ||
4408 | - jmp 6b | ||
4409 | - | ||
4410 | - /* Unlock after loop requires wakeup. */ | ||
4411 | -10: | ||
4412 | -#if cond_lock == 0 | ||
4413 | - movl %ebx, %eax | ||
4414 | -#else | ||
4415 | - leal cond_lock(%ebx), %eax | ||
4416 | -#endif | ||
4417 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4418 | - xorl %ecx, %ecx | ||
4419 | -#endif | ||
4420 | - cmpl $-1, dep_mutex(%ebx) | ||
4421 | - setne %cl | ||
4422 | - subl $1, %ecx | ||
4423 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4424 | -#if LLL_PRIVATE != 0 | ||
4425 | - addl $LLL_PRIVATE, %ecx | ||
4426 | -#endif | ||
4427 | - call __lll_unlock_wake | ||
4428 | - jmp 11b | ||
4429 | - | ||
4430 | -#ifndef __ASSUME_FUTEX_CLOCK_REALTIME | ||
4431 | - cfi_adjust_cfa_offset(-FRAME_SIZE) | ||
4432 | -.Lreltmo: | ||
4433 | - /* Get internal lock. */ | ||
4434 | - movl $1, %edx | ||
4435 | - xorl %eax, %eax | ||
4436 | - LOCK | ||
4437 | -# if cond_lock == 0 | ||
4438 | - cmpxchgl %edx, (%ebx) | ||
4439 | -# else | ||
4440 | - cmpxchgl %edx, cond_lock(%ebx) | ||
4441 | -# endif | ||
4442 | - jnz 101f | ||
4443 | - | ||
4444 | - /* Store the reference to the mutex. If there is already a | ||
4445 | - different value in there this is a bad user bug. */ | ||
4446 | -102: cmpl $-1, dep_mutex(%ebx) | ||
4447 | - movl 24(%esp), %eax | ||
4448 | - je 117f | ||
4449 | - movl %eax, dep_mutex(%ebx) | ||
4450 | - | ||
4451 | - /* Unlock the mutex. */ | ||
4452 | -117: xorl %edx, %edx | ||
4453 | - call __pthread_mutex_unlock_usercnt | ||
4454 | - | ||
4455 | - testl %eax, %eax | ||
4456 | - jne 16b | ||
4457 | - | ||
4458 | - addl $1, total_seq(%ebx) | ||
4459 | - adcl $0, total_seq+4(%ebx) | ||
4460 | - addl $1, cond_futex(%ebx) | ||
4461 | - addl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
4462 | - | ||
4463 | - subl $FRAME_SIZE, %esp | ||
4464 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
4465 | - | ||
4466 | - /* Get and store current wakeup_seq value. */ | ||
4467 | - movl wakeup_seq(%ebx), %edi | ||
4468 | - movl wakeup_seq+4(%ebx), %edx | ||
4469 | - movl broadcast_seq(%ebx), %eax | ||
4470 | - movl %edi, 4(%esp) | ||
4471 | - movl %edx, 8(%esp) | ||
4472 | - movl %eax, 12(%esp) | ||
4473 | - | ||
4474 | - /* Reset the pi-requeued flag. */ | ||
4475 | - movl $0, 16(%esp) | ||
4476 | - | ||
4477 | - /* Get the current time. */ | ||
4478 | -108: movl %ebx, %edx | ||
4479 | -# ifdef __NR_clock_gettime | ||
4480 | - /* Get the clock number. */ | ||
4481 | - movl cond_nwaiters(%ebx), %ebx | ||
4482 | - andl $((1 << nwaiters_shift) - 1), %ebx | ||
4483 | - /* Only clocks 0 and 1 are allowed so far. Both are handled in the | ||
4484 | - kernel. */ | ||
4485 | - leal 24(%esp), %ecx | ||
4486 | - movl $__NR_clock_gettime, %eax | ||
4487 | - ENTER_KERNEL | ||
4488 | - movl %edx, %ebx | ||
4489 | - | ||
4490 | - /* Compute relative timeout. */ | ||
4491 | - movl (%ebp), %ecx | ||
4492 | - movl 4(%ebp), %edx | ||
4493 | - subl 24(%esp), %ecx | ||
4494 | - subl 28(%esp), %edx | ||
4495 | -# else | ||
4496 | - /* Get the current time. */ | ||
4497 | - leal 24(%esp), %ebx | ||
4498 | - xorl %ecx, %ecx | ||
4499 | - movl $__NR_gettimeofday, %eax | ||
4500 | - ENTER_KERNEL | ||
4501 | - movl %edx, %ebx | ||
4502 | - | ||
4503 | - /* Compute relative timeout. */ | ||
4504 | - movl 28(%esp), %eax | ||
4505 | - movl $1000, %edx | ||
4506 | - mul %edx /* Milli seconds to nano seconds. */ | ||
4507 | - movl (%ebp), %ecx | ||
4508 | - movl 4(%ebp), %edx | ||
4509 | - subl 24(%esp), %ecx | ||
4510 | - subl %eax, %edx | ||
4511 | -# endif | ||
4512 | - jns 112f | ||
4513 | - addl $1000000000, %edx | ||
4514 | - subl $1, %ecx | ||
4515 | -112: testl %ecx, %ecx | ||
4516 | - movl $-ETIMEDOUT, %esi | ||
4517 | - js 106f | ||
4518 | - | ||
4519 | - /* Store relative timeout. */ | ||
4520 | -121: movl %ecx, 24(%esp) | ||
4521 | - movl %edx, 28(%esp) | ||
4522 | - | ||
4523 | - movl cond_futex(%ebx), %edi | ||
4524 | - movl %edi, 20(%esp) | ||
4525 | - | ||
4526 | - /* Unlock. */ | ||
4527 | - LOCK | ||
4528 | -# if cond_lock == 0 | ||
4529 | - subl $1, (%ebx) | ||
4530 | -# else | ||
4531 | - subl $1, cond_lock(%ebx) | ||
4532 | -# endif | ||
4533 | - jne 103f | ||
4534 | - | ||
4535 | -.LcleanupSTART2: | ||
4536 | -104: call __pthread_enable_asynccancel | ||
4537 | - movl %eax, (%esp) | ||
4538 | - | ||
4539 | - leal 24(%esp), %esi | ||
4540 | -# if FUTEX_PRIVATE_FLAG > 255 | ||
4541 | - xorl %ecx, %ecx | ||
4542 | -# endif | ||
4543 | - cmpl $-1, dep_mutex(%ebx) | ||
4544 | - sete %cl | ||
4545 | - subl $1, %ecx | ||
4546 | -# ifdef __ASSUME_PRIVATE_FUTEX | ||
4547 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
4548 | -# else | ||
4549 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
4550 | -# endif | ||
4551 | -# if FUTEX_WAIT != 0 | ||
4552 | - addl $FUTEX_WAIT, %ecx | ||
4553 | -# endif | ||
4554 | - movl 20(%esp), %edx | ||
4555 | - addl $cond_futex, %ebx | ||
4556 | -.Ladd_cond_futex2: | ||
4557 | - movl $SYS_futex, %eax | ||
4558 | - ENTER_KERNEL | ||
4559 | - subl $cond_futex, %ebx | ||
4560 | -.Lsub_cond_futex2: | ||
4561 | - movl %eax, %esi | ||
4562 | - | ||
4563 | -141: movl (%esp), %eax | ||
4564 | - call __pthread_disable_asynccancel | ||
4565 | -.LcleanupEND2: | ||
4566 | - | ||
4567 | - | ||
4568 | - /* Lock. */ | ||
4569 | - movl $1, %edx | ||
4570 | - xorl %eax, %eax | ||
4571 | - LOCK | ||
4572 | -# if cond_lock == 0 | ||
4573 | - cmpxchgl %edx, (%ebx) | ||
4574 | -# else | ||
4575 | - cmpxchgl %edx, cond_lock(%ebx) | ||
4576 | -# endif | ||
4577 | - jnz 105f | ||
4578 | - | ||
4579 | -106: movl broadcast_seq(%ebx), %eax | ||
4580 | - cmpl 12(%esp), %eax | ||
4581 | - jne 23b | ||
4582 | - | ||
4583 | - movl woken_seq(%ebx), %eax | ||
4584 | - movl woken_seq+4(%ebx), %ecx | ||
4585 | - | ||
4586 | - movl wakeup_seq(%ebx), %edi | ||
4587 | - movl wakeup_seq+4(%ebx), %edx | ||
4588 | - | ||
4589 | - cmpl 8(%esp), %edx | ||
4590 | - jne 107f | ||
4591 | - cmpl 4(%esp), %edi | ||
4592 | - je 115f | ||
4593 | - | ||
4594 | -107: cmpl %ecx, %edx | ||
4595 | - jne 9b | ||
4596 | - cmp %eax, %edi | ||
4597 | - jne 9b | ||
4598 | - | ||
4599 | -115: cmpl $-ETIMEDOUT, %esi | ||
4600 | - je 28b | ||
4601 | - | ||
4602 | - jmp 8b | ||
4603 | - | ||
4604 | - cfi_adjust_cfa_offset(-FRAME_SIZE) | ||
4605 | - /* Initial locking failed. */ | ||
4606 | -101: | ||
4607 | -# if cond_lock == 0 | ||
4608 | - movl %ebx, %edx | ||
4609 | -# else | ||
4610 | - leal cond_lock(%ebx), %edx | ||
4611 | -# endif | ||
4612 | -# if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4613 | - xorl %ecx, %ecx | ||
4614 | -# endif | ||
4615 | - cmpl $-1, dep_mutex(%ebx) | ||
4616 | - setne %cl | ||
4617 | - subl $1, %ecx | ||
4618 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4619 | -# if LLL_PRIVATE != 0 | ||
4620 | - addl $LLL_PRIVATE, %ecx | ||
4621 | -# endif | ||
4622 | - call __lll_lock_wait | ||
4623 | - jmp 102b | ||
4624 | - | ||
4625 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
4626 | - | ||
4627 | - /* Unlock in loop requires wakeup. */ | ||
4628 | -103: | ||
4629 | -# if cond_lock == 0 | ||
4630 | - movl %ebx, %eax | ||
4631 | -# else | ||
4632 | - leal cond_lock(%ebx), %eax | ||
4633 | -# endif | ||
4634 | -# if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4635 | - xorl %ecx, %ecx | ||
4636 | -# endif | ||
4637 | - cmpl $-1, dep_mutex(%ebx) | ||
4638 | - setne %cl | ||
4639 | - subl $1, %ecx | ||
4640 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4641 | -# if LLL_PRIVATE != 0 | ||
4642 | - addl $LLL_PRIVATE, %ecx | ||
4643 | -# endif | ||
4644 | - call __lll_unlock_wake | ||
4645 | - jmp 104b | ||
4646 | - | ||
4647 | - /* Locking in loop failed. */ | ||
4648 | -105: | ||
4649 | -# if cond_lock == 0 | ||
4650 | - movl %ebx, %edx | ||
4651 | -# else | ||
4652 | - leal cond_lock(%ebx), %edx | ||
4653 | -# endif | ||
4654 | -# if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4655 | - xorl %ecx, %ecx | ||
4656 | -# endif | ||
4657 | - cmpl $-1, dep_mutex(%ebx) | ||
4658 | - setne %cl | ||
4659 | - subl $1, %ecx | ||
4660 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4661 | -# if LLL_PRIVATE != 0 | ||
4662 | - addl $LLL_PRIVATE, %ecx | ||
4663 | -# endif | ||
4664 | - call __lll_lock_wait | ||
4665 | - jmp 106b | ||
4666 | -#endif | ||
4667 | - | ||
4668 | - .size __pthread_cond_timedwait, .-__pthread_cond_timedwait | ||
4669 | -versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, | ||
4670 | - GLIBC_2_3_2) | ||
4671 | - | ||
4672 | - | ||
4673 | - .type __condvar_tw_cleanup2, @function | ||
4674 | -__condvar_tw_cleanup2: | ||
4675 | - subl $cond_futex, %ebx | ||
4676 | - .size __condvar_tw_cleanup2, .-__condvar_tw_cleanup2 | ||
4677 | - .type __condvar_tw_cleanup, @function | ||
4678 | -__condvar_tw_cleanup: | ||
4679 | - movl %eax, %esi | ||
4680 | - | ||
4681 | - /* Get internal lock. */ | ||
4682 | - movl $1, %edx | ||
4683 | - xorl %eax, %eax | ||
4684 | - LOCK | ||
4685 | -#if cond_lock == 0 | ||
4686 | - cmpxchgl %edx, (%ebx) | ||
4687 | -#else | ||
4688 | - cmpxchgl %edx, cond_lock(%ebx) | ||
4689 | -#endif | ||
4690 | - jz 1f | ||
4691 | - | ||
4692 | -#if cond_lock == 0 | ||
4693 | - movl %ebx, %edx | ||
4694 | -#else | ||
4695 | - leal cond_lock(%ebx), %edx | ||
4696 | -#endif | ||
4697 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4698 | - xorl %ecx, %ecx | ||
4699 | -#endif | ||
4700 | - cmpl $-1, dep_mutex(%ebx) | ||
4701 | - setne %cl | ||
4702 | - subl $1, %ecx | ||
4703 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4704 | -#if LLL_PRIVATE != 0 | ||
4705 | - addl $LLL_PRIVATE, %ecx | ||
4706 | -#endif | ||
4707 | - call __lll_lock_wait | ||
4708 | - | ||
4709 | -1: movl broadcast_seq(%ebx), %eax | ||
4710 | - cmpl 12(%esp), %eax | ||
4711 | - jne 3f | ||
4712 | - | ||
4713 | - /* We increment the wakeup_seq counter only if it is lower than | ||
4714 | - total_seq. If this is not the case the thread was woken and | ||
4715 | - then canceled. In this case we ignore the signal. */ | ||
4716 | - movl total_seq(%ebx), %eax | ||
4717 | - movl total_seq+4(%ebx), %edi | ||
4718 | - cmpl wakeup_seq+4(%ebx), %edi | ||
4719 | - jb 6f | ||
4720 | - ja 7f | ||
4721 | - cmpl wakeup_seq(%ebx), %eax | ||
4722 | - jbe 7f | ||
4723 | - | ||
4724 | -6: addl $1, wakeup_seq(%ebx) | ||
4725 | - adcl $0, wakeup_seq+4(%ebx) | ||
4726 | - addl $1, cond_futex(%ebx) | ||
4727 | - | ||
4728 | -7: addl $1, woken_seq(%ebx) | ||
4729 | - adcl $0, woken_seq+4(%ebx) | ||
4730 | - | ||
4731 | -3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
4732 | - | ||
4733 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
4734 | - xorl %edi, %edi | ||
4735 | - movl total_seq(%ebx), %eax | ||
4736 | - andl total_seq+4(%ebx), %eax | ||
4737 | - cmpl $0xffffffff, %eax | ||
4738 | - jne 4f | ||
4739 | - movl cond_nwaiters(%ebx), %eax | ||
4740 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
4741 | - jne 4f | ||
4742 | - | ||
4743 | - addl $cond_nwaiters, %ebx | ||
4744 | - movl $SYS_futex, %eax | ||
4745 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
4746 | - xorl %ecx, %ecx | ||
4747 | -#endif | ||
4748 | - cmpl $-1, dep_mutex-cond_nwaiters(%ebx) | ||
4749 | - sete %cl | ||
4750 | - subl $1, %ecx | ||
4751 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
4752 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
4753 | -#else | ||
4754 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
4755 | -#endif | ||
4756 | - addl $FUTEX_WAKE, %ecx | ||
4757 | - movl $1, %edx | ||
4758 | - ENTER_KERNEL | ||
4759 | - subl $cond_nwaiters, %ebx | ||
4760 | - movl $1, %edi | ||
4761 | - | ||
4762 | -4: LOCK | ||
4763 | -#if cond_lock == 0 | ||
4764 | - subl $1, (%ebx) | ||
4765 | -#else | ||
4766 | - subl $1, cond_lock(%ebx) | ||
4767 | -#endif | ||
4768 | - je 2f | ||
4769 | - | ||
4770 | -#if cond_lock == 0 | ||
4771 | - movl %ebx, %eax | ||
4772 | -#else | ||
4773 | - leal cond_lock(%ebx), %eax | ||
4774 | -#endif | ||
4775 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
4776 | - xorl %ecx, %ecx | ||
4777 | -#endif | ||
4778 | - cmpl $-1, dep_mutex(%ebx) | ||
4779 | - setne %cl | ||
4780 | - subl $1, %ecx | ||
4781 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
4782 | -#if LLL_PRIVATE != 0 | ||
4783 | - addl $LLL_PRIVATE, %ecx | ||
4784 | -#endif | ||
4785 | - call __lll_unlock_wake | ||
4786 | - | ||
4787 | - /* Wake up all waiters to make sure no signal gets lost. */ | ||
4788 | -2: testl %edi, %edi | ||
4789 | - jnz 5f | ||
4790 | - addl $cond_futex, %ebx | ||
4791 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
4792 | - xorl %ecx, %ecx | ||
4793 | -#endif | ||
4794 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
4795 | - sete %cl | ||
4796 | - subl $1, %ecx | ||
4797 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
4798 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
4799 | -#else | ||
4800 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
4801 | -#endif | ||
4802 | - addl $FUTEX_WAKE, %ecx | ||
4803 | - movl $SYS_futex, %eax | ||
4804 | - movl $0x7fffffff, %edx | ||
4805 | - ENTER_KERNEL | ||
4806 | - | ||
4807 | - /* Lock the mutex only if we don't own it already. This only happens | ||
4808 | - in case of PI mutexes, if we got cancelled after a successful | ||
4809 | - return of the futex syscall and before disabling async | ||
4810 | - cancellation. */ | ||
4811 | -5: movl 24+FRAME_SIZE(%esp), %eax | ||
4812 | - movl MUTEX_KIND(%eax), %ebx | ||
4813 | - andl $(ROBUST_BIT|PI_BIT), %ebx | ||
4814 | - cmpl $PI_BIT, %ebx | ||
4815 | - jne 8f | ||
4816 | - | ||
4817 | - movl (%eax), %ebx | ||
4818 | - andl $TID_MASK, %ebx | ||
4819 | - cmpl %ebx, %gs:TID | ||
4820 | - jne 8f | ||
4821 | - /* We managed to get the lock. Fix it up before returning. */ | ||
4822 | - call __pthread_mutex_cond_lock_adjust | ||
4823 | - jmp 9f | ||
4824 | - | ||
4825 | -8: call __pthread_mutex_cond_lock | ||
4826 | - | ||
4827 | -9: movl %esi, (%esp) | ||
4828 | -.LcallUR: | ||
4829 | - call _Unwind_Resume | ||
4830 | - hlt | ||
4831 | -.LENDCODE: | ||
4832 | - cfi_endproc | ||
4833 | - .size __condvar_tw_cleanup, .-__condvar_tw_cleanup | ||
4834 | - | ||
4835 | - | ||
4836 | - .section .gcc_except_table,"a",@progbits | ||
4837 | -.LexceptSTART: | ||
4838 | - .byte DW_EH_PE_omit # @LPStart format (omit) | ||
4839 | - .byte DW_EH_PE_omit # @TType format (omit) | ||
4840 | - .byte DW_EH_PE_sdata4 # call-site format | ||
4841 | - # DW_EH_PE_sdata4 | ||
4842 | - .uleb128 .Lcstend-.Lcstbegin | ||
4843 | -.Lcstbegin: | ||
4844 | - .long .LcleanupSTART-.LSTARTCODE | ||
4845 | - .long .Ladd_cond_futex_pi-.LcleanupSTART | ||
4846 | - .long __condvar_tw_cleanup-.LSTARTCODE | ||
4847 | - .uleb128 0 | ||
4848 | - .long .Ladd_cond_futex_pi-.LSTARTCODE | ||
4849 | - .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi | ||
4850 | - .long __condvar_tw_cleanup2-.LSTARTCODE | ||
4851 | - .uleb128 0 | ||
4852 | - .long .Lsub_cond_futex_pi-.LSTARTCODE | ||
4853 | - .long .Ladd_cond_futex-.Lsub_cond_futex_pi | ||
4854 | - .long __condvar_tw_cleanup-.LSTARTCODE | ||
4855 | - .uleb128 0 | ||
4856 | - .long .Ladd_cond_futex-.LSTARTCODE | ||
4857 | - .long .Lsub_cond_futex-.Ladd_cond_futex | ||
4858 | - .long __condvar_tw_cleanup2-.LSTARTCODE | ||
4859 | - .uleb128 0 | ||
4860 | - .long .Lsub_cond_futex-.LSTARTCODE | ||
4861 | - .long .LcleanupEND-.Lsub_cond_futex | ||
4862 | - .long __condvar_tw_cleanup-.LSTARTCODE | ||
4863 | - .uleb128 0 | ||
4864 | -#ifndef __ASSUME_FUTEX_CLOCK_REALTIME | ||
4865 | - .long .LcleanupSTART2-.LSTARTCODE | ||
4866 | - .long .Ladd_cond_futex2-.LcleanupSTART2 | ||
4867 | - .long __condvar_tw_cleanup-.LSTARTCODE | ||
4868 | - .uleb128 0 | ||
4869 | - .long .Ladd_cond_futex2-.LSTARTCODE | ||
4870 | - .long .Lsub_cond_futex2-.Ladd_cond_futex2 | ||
4871 | - .long __condvar_tw_cleanup2-.LSTARTCODE | ||
4872 | - .uleb128 0 | ||
4873 | - .long .Lsub_cond_futex2-.LSTARTCODE | ||
4874 | - .long .LcleanupEND2-.Lsub_cond_futex2 | ||
4875 | - .long __condvar_tw_cleanup-.LSTARTCODE | ||
4876 | - .uleb128 0 | ||
4877 | -#endif | ||
4878 | - .long .LcallUR-.LSTARTCODE | ||
4879 | - .long .LENDCODE-.LcallUR | ||
4880 | - .long 0 | ||
4881 | - .uleb128 0 | ||
4882 | -.Lcstend: | ||
4883 | - | ||
4884 | - | ||
4885 | -#ifdef SHARED | ||
4886 | - .hidden DW.ref.__gcc_personality_v0 | ||
4887 | - .weak DW.ref.__gcc_personality_v0 | ||
4888 | - .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits | ||
4889 | - .align 4 | ||
4890 | - .type DW.ref.__gcc_personality_v0, @object | ||
4891 | - .size DW.ref.__gcc_personality_v0, 4 | ||
4892 | -DW.ref.__gcc_personality_v0: | ||
4893 | - .long __gcc_personality_v0 | ||
4894 | -#endif | ||
4895 | diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S | ||
4896 | deleted file mode 100644 | ||
4897 | index 5016718..0000000 | ||
4898 | --- a/sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S | ||
4899 | +++ /dev/null | ||
4900 | @@ -1,642 +0,0 @@ | ||
4901 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
4902 | - This file is part of the GNU C Library. | ||
4903 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
4904 | - | ||
4905 | - The GNU C Library is free software; you can redistribute it and/or | ||
4906 | - modify it under the terms of the GNU Lesser General Public | ||
4907 | - License as published by the Free Software Foundation; either | ||
4908 | - version 2.1 of the License, or (at your option) any later version. | ||
4909 | - | ||
4910 | - The GNU C Library is distributed in the hope that it will be useful, | ||
4911 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
4912 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
4913 | - Lesser General Public License for more details. | ||
4914 | - | ||
4915 | - You should have received a copy of the GNU Lesser General Public | ||
4916 | - License along with the GNU C Library; if not, see | ||
4917 | - <http://www.gnu.org/licenses/>. */ | ||
4918 | - | ||
4919 | -#include <sysdep.h> | ||
4920 | -#include <shlib-compat.h> | ||
4921 | -#include <lowlevellock.h> | ||
4922 | -#include <lowlevelcond.h> | ||
4923 | -#include <tcb-offsets.h> | ||
4924 | -#include <pthread-errnos.h> | ||
4925 | -#include <pthread-pi-defines.h> | ||
4926 | -#include <kernel-features.h> | ||
4927 | -#include <stap-probe.h> | ||
4928 | - | ||
4929 | - | ||
4930 | - .text | ||
4931 | - | ||
4932 | -/* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */ | ||
4933 | - .globl __pthread_cond_wait | ||
4934 | - .type __pthread_cond_wait, @function | ||
4935 | - .align 16 | ||
4936 | -__pthread_cond_wait: | ||
4937 | -.LSTARTCODE: | ||
4938 | - cfi_startproc | ||
4939 | -#ifdef SHARED | ||
4940 | - cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, | ||
4941 | - DW.ref.__gcc_personality_v0) | ||
4942 | - cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) | ||
4943 | -#else | ||
4944 | - cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) | ||
4945 | - cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) | ||
4946 | -#endif | ||
4947 | - | ||
4948 | - pushl %ebp | ||
4949 | - cfi_adjust_cfa_offset(4) | ||
4950 | - cfi_rel_offset(%ebp, 0) | ||
4951 | - pushl %edi | ||
4952 | - cfi_adjust_cfa_offset(4) | ||
4953 | - cfi_rel_offset(%edi, 0) | ||
4954 | - pushl %esi | ||
4955 | - cfi_adjust_cfa_offset(4) | ||
4956 | - cfi_rel_offset(%esi, 0) | ||
4957 | - pushl %ebx | ||
4958 | - cfi_adjust_cfa_offset(4) | ||
4959 | - cfi_rel_offset(%ebx, 0) | ||
4960 | - | ||
4961 | - xorl %esi, %esi | ||
4962 | - movl 20(%esp), %ebx | ||
4963 | - | ||
4964 | - LIBC_PROBE (cond_wait, 2, 24(%esp), %ebx) | ||
4965 | - | ||
4966 | - /* Get internal lock. */ | ||
4967 | - movl $1, %edx | ||
4968 | - xorl %eax, %eax | ||
4969 | - LOCK | ||
4970 | -#if cond_lock == 0 | ||
4971 | - cmpxchgl %edx, (%ebx) | ||
4972 | -#else | ||
4973 | - cmpxchgl %edx, cond_lock(%ebx) | ||
4974 | -#endif | ||
4975 | - jnz 1f | ||
4976 | - | ||
4977 | - /* Store the reference to the mutex. If there is already a | ||
4978 | - different value in there this is a bad user bug. */ | ||
4979 | -2: cmpl $-1, dep_mutex(%ebx) | ||
4980 | - movl 24(%esp), %eax | ||
4981 | - je 15f | ||
4982 | - movl %eax, dep_mutex(%ebx) | ||
4983 | - | ||
4984 | - /* Unlock the mutex. */ | ||
4985 | -15: xorl %edx, %edx | ||
4986 | - call __pthread_mutex_unlock_usercnt | ||
4987 | - | ||
4988 | - testl %eax, %eax | ||
4989 | - jne 12f | ||
4990 | - | ||
4991 | - addl $1, total_seq(%ebx) | ||
4992 | - adcl $0, total_seq+4(%ebx) | ||
4993 | - addl $1, cond_futex(%ebx) | ||
4994 | - addl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
4995 | - | ||
4996 | -#define FRAME_SIZE 20 | ||
4997 | - subl $FRAME_SIZE, %esp | ||
4998 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
4999 | - cfi_remember_state | ||
5000 | - | ||
5001 | - /* Get and store current wakeup_seq value. */ | ||
5002 | - movl wakeup_seq(%ebx), %edi | ||
5003 | - movl wakeup_seq+4(%ebx), %edx | ||
5004 | - movl broadcast_seq(%ebx), %eax | ||
5005 | - movl %edi, 4(%esp) | ||
5006 | - movl %edx, 8(%esp) | ||
5007 | - movl %eax, 12(%esp) | ||
5008 | - | ||
5009 | - /* Reset the pi-requeued flag. */ | ||
5010 | -8: movl $0, 16(%esp) | ||
5011 | - movl cond_futex(%ebx), %ebp | ||
5012 | - | ||
5013 | - /* Unlock. */ | ||
5014 | - LOCK | ||
5015 | -#if cond_lock == 0 | ||
5016 | - subl $1, (%ebx) | ||
5017 | -#else | ||
5018 | - subl $1, cond_lock(%ebx) | ||
5019 | -#endif | ||
5020 | - jne 3f | ||
5021 | - | ||
5022 | -.LcleanupSTART: | ||
5023 | -4: call __pthread_enable_asynccancel | ||
5024 | - movl %eax, (%esp) | ||
5025 | - | ||
5026 | - xorl %ecx, %ecx | ||
5027 | - cmpl $-1, dep_mutex(%ebx) | ||
5028 | - sete %cl | ||
5029 | - je 18f | ||
5030 | - | ||
5031 | - movl dep_mutex(%ebx), %edi | ||
5032 | - /* Requeue to a non-robust PI mutex if the PI bit is set and | ||
5033 | - the robust bit is not set. */ | ||
5034 | - movl MUTEX_KIND(%edi), %eax | ||
5035 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
5036 | - cmpl $PI_BIT, %eax | ||
5037 | - jne 18f | ||
5038 | - | ||
5039 | - movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx | ||
5040 | - movl %ebp, %edx | ||
5041 | - xorl %esi, %esi | ||
5042 | - addl $cond_futex, %ebx | ||
5043 | -.Ladd_cond_futex_pi: | ||
5044 | - movl $SYS_futex, %eax | ||
5045 | - ENTER_KERNEL | ||
5046 | - subl $cond_futex, %ebx | ||
5047 | -.Lsub_cond_futex_pi: | ||
5048 | - /* Set the pi-requeued flag only if the kernel has returned 0. The | ||
5049 | - kernel does not hold the mutex on error. */ | ||
5050 | - cmpl $0, %eax | ||
5051 | - sete 16(%esp) | ||
5052 | - je 19f | ||
5053 | - | ||
5054 | - /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns | ||
5055 | - successfully, it has already locked the mutex for us and the | ||
5056 | - pi_flag (16(%esp)) is set to denote that fact. However, if another | ||
5057 | - thread changed the futex value before we entered the wait, the | ||
5058 | - syscall may return an EAGAIN and the mutex is not locked. We go | ||
5059 | - ahead with a success anyway since later we look at the pi_flag to | ||
5060 | - decide if we got the mutex or not. The sequence numbers then make | ||
5061 | - sure that only one of the threads actually wake up. We retry using | ||
5062 | - normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal | ||
5063 | - and PI futexes don't mix. | ||
5064 | - | ||
5065 | - Note that we don't check for EAGAIN specifically; we assume that the | ||
5066 | - only other error the futex function could return is EAGAIN since | ||
5067 | - anything else would mean an error in our function. It is too | ||
5068 | - expensive to do that check for every call (which is quite common in | ||
5069 | - case of a large number of threads), so it has been skipped. */ | ||
5070 | - cmpl $-ENOSYS, %eax | ||
5071 | - jne 19f | ||
5072 | - xorl %ecx, %ecx | ||
5073 | - | ||
5074 | -18: subl $1, %ecx | ||
5075 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5076 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
5077 | -#else | ||
5078 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
5079 | -#endif | ||
5080 | -#if FUTEX_WAIT != 0 | ||
5081 | - addl $FUTEX_WAIT, %ecx | ||
5082 | -#endif | ||
5083 | - movl %ebp, %edx | ||
5084 | - addl $cond_futex, %ebx | ||
5085 | -.Ladd_cond_futex: | ||
5086 | - movl $SYS_futex, %eax | ||
5087 | - ENTER_KERNEL | ||
5088 | - subl $cond_futex, %ebx | ||
5089 | -.Lsub_cond_futex: | ||
5090 | - | ||
5091 | -19: movl (%esp), %eax | ||
5092 | - call __pthread_disable_asynccancel | ||
5093 | -.LcleanupEND: | ||
5094 | - | ||
5095 | - /* Lock. */ | ||
5096 | - movl $1, %edx | ||
5097 | - xorl %eax, %eax | ||
5098 | - LOCK | ||
5099 | -#if cond_lock == 0 | ||
5100 | - cmpxchgl %edx, (%ebx) | ||
5101 | -#else | ||
5102 | - cmpxchgl %edx, cond_lock(%ebx) | ||
5103 | -#endif | ||
5104 | - jnz 5f | ||
5105 | - | ||
5106 | -6: movl broadcast_seq(%ebx), %eax | ||
5107 | - cmpl 12(%esp), %eax | ||
5108 | - jne 16f | ||
5109 | - | ||
5110 | - movl woken_seq(%ebx), %eax | ||
5111 | - movl woken_seq+4(%ebx), %ecx | ||
5112 | - | ||
5113 | - movl wakeup_seq(%ebx), %edi | ||
5114 | - movl wakeup_seq+4(%ebx), %edx | ||
5115 | - | ||
5116 | - cmpl 8(%esp), %edx | ||
5117 | - jne 7f | ||
5118 | - cmpl 4(%esp), %edi | ||
5119 | - je 22f | ||
5120 | - | ||
5121 | -7: cmpl %ecx, %edx | ||
5122 | - jne 9f | ||
5123 | - cmp %eax, %edi | ||
5124 | - je 22f | ||
5125 | - | ||
5126 | -9: addl $1, woken_seq(%ebx) | ||
5127 | - adcl $0, woken_seq+4(%ebx) | ||
5128 | - | ||
5129 | - /* Unlock */ | ||
5130 | -16: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
5131 | - | ||
5132 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
5133 | - movl total_seq(%ebx), %eax | ||
5134 | - andl total_seq+4(%ebx), %eax | ||
5135 | - cmpl $0xffffffff, %eax | ||
5136 | - jne 17f | ||
5137 | - movl cond_nwaiters(%ebx), %eax | ||
5138 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
5139 | - jne 17f | ||
5140 | - | ||
5141 | - addl $cond_nwaiters, %ebx | ||
5142 | - movl $SYS_futex, %eax | ||
5143 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
5144 | - xorl %ecx, %ecx | ||
5145 | -#endif | ||
5146 | - cmpl $-1, dep_mutex-cond_nwaiters(%ebx) | ||
5147 | - sete %cl | ||
5148 | - subl $1, %ecx | ||
5149 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5150 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
5151 | -#else | ||
5152 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
5153 | -#endif | ||
5154 | - addl $FUTEX_WAKE, %ecx | ||
5155 | - movl $1, %edx | ||
5156 | - ENTER_KERNEL | ||
5157 | - subl $cond_nwaiters, %ebx | ||
5158 | - | ||
5159 | -17: LOCK | ||
5160 | -#if cond_lock == 0 | ||
5161 | - subl $1, (%ebx) | ||
5162 | -#else | ||
5163 | - subl $1, cond_lock(%ebx) | ||
5164 | -#endif | ||
5165 | - jne 10f | ||
5166 | - | ||
5167 | - /* With requeue_pi, the mutex lock is held in the kernel. */ | ||
5168 | -11: movl 24+FRAME_SIZE(%esp), %eax | ||
5169 | - movl 16(%esp), %ecx | ||
5170 | - testl %ecx, %ecx | ||
5171 | - jnz 21f | ||
5172 | - | ||
5173 | - call __pthread_mutex_cond_lock | ||
5174 | -20: addl $FRAME_SIZE, %esp | ||
5175 | - cfi_adjust_cfa_offset(-FRAME_SIZE); | ||
5176 | - | ||
5177 | -14: popl %ebx | ||
5178 | - cfi_adjust_cfa_offset(-4) | ||
5179 | - cfi_restore(%ebx) | ||
5180 | - popl %esi | ||
5181 | - cfi_adjust_cfa_offset(-4) | ||
5182 | - cfi_restore(%esi) | ||
5183 | - popl %edi | ||
5184 | - cfi_adjust_cfa_offset(-4) | ||
5185 | - cfi_restore(%edi) | ||
5186 | - popl %ebp | ||
5187 | - cfi_adjust_cfa_offset(-4) | ||
5188 | - cfi_restore(%ebp) | ||
5189 | - | ||
5190 | - /* We return the result of the mutex_lock operation. */ | ||
5191 | - ret | ||
5192 | - | ||
5193 | - cfi_restore_state | ||
5194 | - | ||
5195 | -21: call __pthread_mutex_cond_lock_adjust | ||
5196 | - xorl %eax, %eax | ||
5197 | - jmp 20b | ||
5198 | - | ||
5199 | - cfi_adjust_cfa_offset(-FRAME_SIZE); | ||
5200 | - | ||
5201 | - /* We need to go back to futex_wait. If we're using requeue_pi, then | ||
5202 | - release the mutex we had acquired and go back. */ | ||
5203 | -22: movl 16(%esp), %edx | ||
5204 | - test %edx, %edx | ||
5205 | - jz 8b | ||
5206 | - | ||
5207 | - /* Adjust the mutex values first and then unlock it. The unlock | ||
5208 | - should always succeed or else the kernel did not lock the mutex | ||
5209 | - correctly. */ | ||
5210 | - movl dep_mutex(%ebx), %eax | ||
5211 | - call __pthread_mutex_cond_lock_adjust | ||
5212 | - movl dep_mutex(%ebx), %eax | ||
5213 | - xorl %edx, %edx | ||
5214 | - call __pthread_mutex_unlock_usercnt | ||
5215 | - jmp 8b | ||
5216 | - | ||
5217 | - /* Initial locking failed. */ | ||
5218 | -1: | ||
5219 | -#if cond_lock == 0 | ||
5220 | - movl %ebx, %edx | ||
5221 | -#else | ||
5222 | - leal cond_lock(%ebx), %edx | ||
5223 | -#endif | ||
5224 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5225 | - xorl %ecx, %ecx | ||
5226 | -#endif | ||
5227 | - cmpl $-1, dep_mutex(%ebx) | ||
5228 | - setne %cl | ||
5229 | - subl $1, %ecx | ||
5230 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5231 | -#if LLL_PRIVATE != 0 | ||
5232 | - addl $LLL_PRIVATE, %ecx | ||
5233 | -#endif | ||
5234 | - call __lll_lock_wait | ||
5235 | - jmp 2b | ||
5236 | - | ||
5237 | - /* The initial unlocking of the mutex failed. */ | ||
5238 | -12: | ||
5239 | - LOCK | ||
5240 | -#if cond_lock == 0 | ||
5241 | - subl $1, (%ebx) | ||
5242 | -#else | ||
5243 | - subl $1, cond_lock(%ebx) | ||
5244 | -#endif | ||
5245 | - jne 14b | ||
5246 | - | ||
5247 | - movl %eax, %esi | ||
5248 | -#if cond_lock == 0 | ||
5249 | - movl %ebx, %eax | ||
5250 | -#else | ||
5251 | - leal cond_lock(%ebx), %eax | ||
5252 | -#endif | ||
5253 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5254 | - xorl %ecx, %ecx | ||
5255 | -#endif | ||
5256 | - cmpl $-1, dep_mutex(%ebx) | ||
5257 | - setne %cl | ||
5258 | - subl $1, %ecx | ||
5259 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5260 | -#if LLL_PRIVATE != 0 | ||
5261 | - addl $LLL_PRIVATE, %ecx | ||
5262 | -#endif | ||
5263 | - call __lll_unlock_wake | ||
5264 | - | ||
5265 | - movl %esi, %eax | ||
5266 | - jmp 14b | ||
5267 | - | ||
5268 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
5269 | - | ||
5270 | - /* Unlock in loop requires wakeup. */ | ||
5271 | -3: | ||
5272 | -#if cond_lock == 0 | ||
5273 | - movl %ebx, %eax | ||
5274 | -#else | ||
5275 | - leal cond_lock(%ebx), %eax | ||
5276 | -#endif | ||
5277 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5278 | - xorl %ecx, %ecx | ||
5279 | -#endif | ||
5280 | - cmpl $-1, dep_mutex(%ebx) | ||
5281 | - setne %cl | ||
5282 | - subl $1, %ecx | ||
5283 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5284 | -#if LLL_PRIVATE != 0 | ||
5285 | - addl $LLL_PRIVATE, %ecx | ||
5286 | -#endif | ||
5287 | - call __lll_unlock_wake | ||
5288 | - jmp 4b | ||
5289 | - | ||
5290 | - /* Locking in loop failed. */ | ||
5291 | -5: | ||
5292 | -#if cond_lock == 0 | ||
5293 | - movl %ebx, %edx | ||
5294 | -#else | ||
5295 | - leal cond_lock(%ebx), %edx | ||
5296 | -#endif | ||
5297 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5298 | - xorl %ecx, %ecx | ||
5299 | -#endif | ||
5300 | - cmpl $-1, dep_mutex(%ebx) | ||
5301 | - setne %cl | ||
5302 | - subl $1, %ecx | ||
5303 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5304 | -#if LLL_PRIVATE != 0 | ||
5305 | - addl $LLL_PRIVATE, %ecx | ||
5306 | -#endif | ||
5307 | - call __lll_lock_wait | ||
5308 | - jmp 6b | ||
5309 | - | ||
5310 | - /* Unlock after loop requires wakeup. */ | ||
5311 | -10: | ||
5312 | -#if cond_lock == 0 | ||
5313 | - movl %ebx, %eax | ||
5314 | -#else | ||
5315 | - leal cond_lock(%ebx), %eax | ||
5316 | -#endif | ||
5317 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5318 | - xorl %ecx, %ecx | ||
5319 | -#endif | ||
5320 | - cmpl $-1, dep_mutex(%ebx) | ||
5321 | - setne %cl | ||
5322 | - subl $1, %ecx | ||
5323 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5324 | -#if LLL_PRIVATE != 0 | ||
5325 | - addl $LLL_PRIVATE, %ecx | ||
5326 | -#endif | ||
5327 | - call __lll_unlock_wake | ||
5328 | - jmp 11b | ||
5329 | - | ||
5330 | - .size __pthread_cond_wait, .-__pthread_cond_wait | ||
5331 | -versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, | ||
5332 | - GLIBC_2_3_2) | ||
5333 | - | ||
5334 | - | ||
5335 | - .type __condvar_w_cleanup2, @function | ||
5336 | -__condvar_w_cleanup2: | ||
5337 | - subl $cond_futex, %ebx | ||
5338 | - .size __condvar_w_cleanup2, .-__condvar_w_cleanup2 | ||
5339 | -.LSbl4: | ||
5340 | - .type __condvar_w_cleanup, @function | ||
5341 | -__condvar_w_cleanup: | ||
5342 | - movl %eax, %esi | ||
5343 | - | ||
5344 | - /* Get internal lock. */ | ||
5345 | - movl $1, %edx | ||
5346 | - xorl %eax, %eax | ||
5347 | - LOCK | ||
5348 | -#if cond_lock == 0 | ||
5349 | - cmpxchgl %edx, (%ebx) | ||
5350 | -#else | ||
5351 | - cmpxchgl %edx, cond_lock(%ebx) | ||
5352 | -#endif | ||
5353 | - jz 1f | ||
5354 | - | ||
5355 | -#if cond_lock == 0 | ||
5356 | - movl %ebx, %edx | ||
5357 | -#else | ||
5358 | - leal cond_lock(%ebx), %edx | ||
5359 | -#endif | ||
5360 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5361 | - xorl %ecx, %ecx | ||
5362 | -#endif | ||
5363 | - cmpl $-1, dep_mutex(%ebx) | ||
5364 | - setne %cl | ||
5365 | - subl $1, %ecx | ||
5366 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5367 | -#if LLL_PRIVATE != 0 | ||
5368 | - addl $LLL_PRIVATE, %ecx | ||
5369 | -#endif | ||
5370 | - call __lll_lock_wait | ||
5371 | - | ||
5372 | -1: movl broadcast_seq(%ebx), %eax | ||
5373 | - cmpl 12(%esp), %eax | ||
5374 | - jne 3f | ||
5375 | - | ||
5376 | - /* We increment the wakeup_seq counter only if it is lower than | ||
5377 | - total_seq. If this is not the case the thread was woken and | ||
5378 | - then canceled. In this case we ignore the signal. */ | ||
5379 | - movl total_seq(%ebx), %eax | ||
5380 | - movl total_seq+4(%ebx), %edi | ||
5381 | - cmpl wakeup_seq+4(%ebx), %edi | ||
5382 | - jb 6f | ||
5383 | - ja 7f | ||
5384 | - cmpl wakeup_seq(%ebx), %eax | ||
5385 | - jbe 7f | ||
5386 | - | ||
5387 | -6: addl $1, wakeup_seq(%ebx) | ||
5388 | - adcl $0, wakeup_seq+4(%ebx) | ||
5389 | - addl $1, cond_futex(%ebx) | ||
5390 | - | ||
5391 | -7: addl $1, woken_seq(%ebx) | ||
5392 | - adcl $0, woken_seq+4(%ebx) | ||
5393 | - | ||
5394 | -3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx) | ||
5395 | - | ||
5396 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
5397 | - xorl %edi, %edi | ||
5398 | - movl total_seq(%ebx), %eax | ||
5399 | - andl total_seq+4(%ebx), %eax | ||
5400 | - cmpl $0xffffffff, %eax | ||
5401 | - jne 4f | ||
5402 | - movl cond_nwaiters(%ebx), %eax | ||
5403 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
5404 | - jne 4f | ||
5405 | - | ||
5406 | - addl $cond_nwaiters, %ebx | ||
5407 | - movl $SYS_futex, %eax | ||
5408 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
5409 | - xorl %ecx, %ecx | ||
5410 | -#endif | ||
5411 | - cmpl $-1, dep_mutex-cond_nwaiters(%ebx) | ||
5412 | - sete %cl | ||
5413 | - subl $1, %ecx | ||
5414 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5415 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
5416 | -#else | ||
5417 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
5418 | -#endif | ||
5419 | - addl $FUTEX_WAKE, %ecx | ||
5420 | - movl $1, %edx | ||
5421 | - ENTER_KERNEL | ||
5422 | - subl $cond_nwaiters, %ebx | ||
5423 | - movl $1, %edi | ||
5424 | - | ||
5425 | -4: LOCK | ||
5426 | -#if cond_lock == 0 | ||
5427 | - subl $1, (%ebx) | ||
5428 | -#else | ||
5429 | - subl $1, cond_lock(%ebx) | ||
5430 | -#endif | ||
5431 | - je 2f | ||
5432 | - | ||
5433 | -#if cond_lock == 0 | ||
5434 | - movl %ebx, %eax | ||
5435 | -#else | ||
5436 | - leal cond_lock(%ebx), %eax | ||
5437 | -#endif | ||
5438 | -#if (LLL_SHARED-LLL_PRIVATE) > 255 | ||
5439 | - xorl %ecx, %ecx | ||
5440 | -#endif | ||
5441 | - cmpl $-1, dep_mutex(%ebx) | ||
5442 | - setne %cl | ||
5443 | - subl $1, %ecx | ||
5444 | - andl $(LLL_SHARED-LLL_PRIVATE), %ecx | ||
5445 | -#if LLL_PRIVATE != 0 | ||
5446 | - addl $LLL_PRIVATE, %ecx | ||
5447 | -#endif | ||
5448 | - call __lll_unlock_wake | ||
5449 | - | ||
5450 | - /* Wake up all waiters to make sure no signal gets lost. */ | ||
5451 | -2: testl %edi, %edi | ||
5452 | - jnz 5f | ||
5453 | - addl $cond_futex, %ebx | ||
5454 | -#if FUTEX_PRIVATE_FLAG > 255 | ||
5455 | - xorl %ecx, %ecx | ||
5456 | -#endif | ||
5457 | - cmpl $-1, dep_mutex-cond_futex(%ebx) | ||
5458 | - sete %cl | ||
5459 | - subl $1, %ecx | ||
5460 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5461 | - andl $FUTEX_PRIVATE_FLAG, %ecx | ||
5462 | -#else | ||
5463 | - andl %gs:PRIVATE_FUTEX, %ecx | ||
5464 | -#endif | ||
5465 | - addl $FUTEX_WAKE, %ecx | ||
5466 | - movl $SYS_futex, %eax | ||
5467 | - movl $0x7fffffff, %edx | ||
5468 | - ENTER_KERNEL | ||
5469 | - | ||
5470 | - /* Lock the mutex only if we don't own it already. This only happens | ||
5471 | - in case of PI mutexes, if we got cancelled after a successful | ||
5472 | - return of the futex syscall and before disabling async | ||
5473 | - cancellation. */ | ||
5474 | -5: movl 24+FRAME_SIZE(%esp), %eax | ||
5475 | - movl MUTEX_KIND(%eax), %ebx | ||
5476 | - andl $(ROBUST_BIT|PI_BIT), %ebx | ||
5477 | - cmpl $PI_BIT, %ebx | ||
5478 | - jne 8f | ||
5479 | - | ||
5480 | - movl (%eax), %ebx | ||
5481 | - andl $TID_MASK, %ebx | ||
5482 | - cmpl %ebx, %gs:TID | ||
5483 | - jne 8f | ||
5484 | - /* We managed to get the lock. Fix it up before returning. */ | ||
5485 | - call __pthread_mutex_cond_lock_adjust | ||
5486 | - jmp 9f | ||
5487 | - | ||
5488 | -8: call __pthread_mutex_cond_lock | ||
5489 | - | ||
5490 | -9: movl %esi, (%esp) | ||
5491 | -.LcallUR: | ||
5492 | - call _Unwind_Resume | ||
5493 | - hlt | ||
5494 | -.LENDCODE: | ||
5495 | - cfi_endproc | ||
5496 | - .size __condvar_w_cleanup, .-__condvar_w_cleanup | ||
5497 | - | ||
5498 | - | ||
5499 | - .section .gcc_except_table,"a",@progbits | ||
5500 | -.LexceptSTART: | ||
5501 | - .byte DW_EH_PE_omit # @LPStart format (omit) | ||
5502 | - .byte DW_EH_PE_omit # @TType format (omit) | ||
5503 | - .byte DW_EH_PE_sdata4 # call-site format | ||
5504 | - # DW_EH_PE_sdata4 | ||
5505 | - .uleb128 .Lcstend-.Lcstbegin | ||
5506 | -.Lcstbegin: | ||
5507 | - .long .LcleanupSTART-.LSTARTCODE | ||
5508 | - .long .Ladd_cond_futex_pi-.LcleanupSTART | ||
5509 | - .long __condvar_w_cleanup-.LSTARTCODE | ||
5510 | - .uleb128 0 | ||
5511 | - .long .Ladd_cond_futex_pi-.LSTARTCODE | ||
5512 | - .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi | ||
5513 | - .long __condvar_w_cleanup2-.LSTARTCODE | ||
5514 | - .uleb128 0 | ||
5515 | - .long .Lsub_cond_futex_pi-.LSTARTCODE | ||
5516 | - .long .Ladd_cond_futex-.Lsub_cond_futex_pi | ||
5517 | - .long __condvar_w_cleanup-.LSTARTCODE | ||
5518 | - .uleb128 0 | ||
5519 | - .long .Ladd_cond_futex-.LSTARTCODE | ||
5520 | - .long .Lsub_cond_futex-.Ladd_cond_futex | ||
5521 | - .long __condvar_w_cleanup2-.LSTARTCODE | ||
5522 | - .uleb128 0 | ||
5523 | - .long .Lsub_cond_futex-.LSTARTCODE | ||
5524 | - .long .LcleanupEND-.Lsub_cond_futex | ||
5525 | - .long __condvar_w_cleanup-.LSTARTCODE | ||
5526 | - .uleb128 0 | ||
5527 | - .long .LcallUR-.LSTARTCODE | ||
5528 | - .long .LENDCODE-.LcallUR | ||
5529 | - .long 0 | ||
5530 | - .uleb128 0 | ||
5531 | -.Lcstend: | ||
5532 | - | ||
5533 | -#ifdef SHARED | ||
5534 | - .hidden DW.ref.__gcc_personality_v0 | ||
5535 | - .weak DW.ref.__gcc_personality_v0 | ||
5536 | - .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits | ||
5537 | - .align 4 | ||
5538 | - .type DW.ref.__gcc_personality_v0, @object | ||
5539 | - .size DW.ref.__gcc_personality_v0, 4 | ||
5540 | -DW.ref.__gcc_personality_v0: | ||
5541 | - .long __gcc_personality_v0 | ||
5542 | -#endif | ||
5543 | diff --git a/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h | ||
5544 | index 345e79a..371bc3c 100644 | ||
5545 | --- a/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h | ||
5546 | +++ b/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h | ||
5547 | @@ -123,19 +123,32 @@ typedef union | ||
5548 | |||
5549 | |||
5550 | /* Data structure for conditional variable handling. The structure of | ||
5551 | - the attribute type is deliberately not exposed. */ | ||
5552 | + the attribute type is not exposed on purpose. */ | ||
5553 | typedef union | ||
5554 | { | ||
5555 | struct | ||
5556 | { | ||
5557 | - int __lock; | ||
5558 | - unsigned int __futex; | ||
5559 | - __extension__ unsigned long long int __total_seq; | ||
5560 | - __extension__ unsigned long long int __wakeup_seq; | ||
5561 | - __extension__ unsigned long long int __woken_seq; | ||
5562 | - void *__mutex; | ||
5563 | - unsigned int __nwaiters; | ||
5564 | - unsigned int __broadcast_seq; | ||
5565 | + __extension__ union | ||
5566 | + { | ||
5567 | + __extension__ unsigned long long int __wseq; | ||
5568 | + struct { | ||
5569 | + unsigned int __low; | ||
5570 | + unsigned int __high; | ||
5571 | + } __wseq32; | ||
5572 | + }; | ||
5573 | + __extension__ union | ||
5574 | + { | ||
5575 | + __extension__ unsigned long long int __g1_start; | ||
5576 | + struct { | ||
5577 | + unsigned int __low; | ||
5578 | + unsigned int __high; | ||
5579 | + } __g1_start32; | ||
5580 | + }; | ||
5581 | + unsigned int __g_refs[2]; | ||
5582 | + unsigned int __g_size[2]; | ||
5583 | + unsigned int __g1_orig_size; | ||
5584 | + unsigned int __wrefs; | ||
5585 | + unsigned int __g_signals[2]; | ||
5586 | } __data; | ||
5587 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
5588 | __extension__ long long int __align; | ||
5589 | diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S | ||
5590 | deleted file mode 100644 | ||
5591 | index de455dd..0000000 | ||
5592 | --- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S | ||
5593 | +++ /dev/null | ||
5594 | @@ -1,177 +0,0 @@ | ||
5595 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
5596 | - This file is part of the GNU C Library. | ||
5597 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
5598 | - | ||
5599 | - The GNU C Library is free software; you can redistribute it and/or | ||
5600 | - modify it under the terms of the GNU Lesser General Public | ||
5601 | - License as published by the Free Software Foundation; either | ||
5602 | - version 2.1 of the License, or (at your option) any later version. | ||
5603 | - | ||
5604 | - The GNU C Library is distributed in the hope that it will be useful, | ||
5605 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
5606 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
5607 | - Lesser General Public License for more details. | ||
5608 | - | ||
5609 | - You should have received a copy of the GNU Lesser General Public | ||
5610 | - License along with the GNU C Library; if not, see | ||
5611 | - <http://www.gnu.org/licenses/>. */ | ||
5612 | - | ||
5613 | -#include <sysdep.h> | ||
5614 | -#include <shlib-compat.h> | ||
5615 | -#include <lowlevellock.h> | ||
5616 | -#include <lowlevelcond.h> | ||
5617 | -#include <kernel-features.h> | ||
5618 | -#include <pthread-pi-defines.h> | ||
5619 | -#include <pthread-errnos.h> | ||
5620 | -#include <stap-probe.h> | ||
5621 | - | ||
5622 | - .text | ||
5623 | - | ||
5624 | - /* int pthread_cond_broadcast (pthread_cond_t *cond) */ | ||
5625 | -ENTRY(__pthread_cond_broadcast) | ||
5626 | - | ||
5627 | - LIBC_PROBE (cond_broadcast, 1, %rdi) | ||
5628 | - | ||
5629 | - /* Get internal lock. */ | ||
5630 | - movl $1, %esi | ||
5631 | - xorl %eax, %eax | ||
5632 | - LOCK | ||
5633 | -#if cond_lock == 0 | ||
5634 | - cmpxchgl %esi, (%rdi) | ||
5635 | -#else | ||
5636 | - cmpxchgl %esi, cond_lock(%rdi) | ||
5637 | -#endif | ||
5638 | - jnz 1f | ||
5639 | - | ||
5640 | -2: addq $cond_futex, %rdi | ||
5641 | - movq total_seq-cond_futex(%rdi), %r9 | ||
5642 | - cmpq wakeup_seq-cond_futex(%rdi), %r9 | ||
5643 | - jna 4f | ||
5644 | - | ||
5645 | - /* Cause all currently waiting threads to recognize they are | ||
5646 | - woken up. */ | ||
5647 | - movq %r9, wakeup_seq-cond_futex(%rdi) | ||
5648 | - movq %r9, woken_seq-cond_futex(%rdi) | ||
5649 | - addq %r9, %r9 | ||
5650 | - movl %r9d, (%rdi) | ||
5651 | - incl broadcast_seq-cond_futex(%rdi) | ||
5652 | - | ||
5653 | - /* Get the address of the mutex used. */ | ||
5654 | - mov dep_mutex-cond_futex(%rdi), %R8_LP | ||
5655 | - | ||
5656 | - /* Unlock. */ | ||
5657 | - LOCK | ||
5658 | - decl cond_lock-cond_futex(%rdi) | ||
5659 | - jne 7f | ||
5660 | - | ||
5661 | -8: cmp $-1, %R8_LP | ||
5662 | - je 9f | ||
5663 | - | ||
5664 | - /* Do not use requeue for pshared condvars. */ | ||
5665 | - testl $PS_BIT, MUTEX_KIND(%r8) | ||
5666 | - jne 9f | ||
5667 | - | ||
5668 | - /* Requeue to a PI mutex if the PI bit is set. */ | ||
5669 | - movl MUTEX_KIND(%r8), %eax | ||
5670 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
5671 | - cmpl $PI_BIT, %eax | ||
5672 | - je 81f | ||
5673 | - | ||
5674 | - /* Wake up all threads. */ | ||
5675 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5676 | - movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi | ||
5677 | -#else | ||
5678 | - movl %fs:PRIVATE_FUTEX, %esi | ||
5679 | - orl $FUTEX_CMP_REQUEUE, %esi | ||
5680 | -#endif | ||
5681 | - movl $SYS_futex, %eax | ||
5682 | - movl $1, %edx | ||
5683 | - movl $0x7fffffff, %r10d | ||
5684 | - syscall | ||
5685 | - | ||
5686 | - /* For any kind of error, which mainly is EAGAIN, we try again | ||
5687 | - with WAKE. The general test also covers running on old | ||
5688 | - kernels. */ | ||
5689 | - cmpq $-4095, %rax | ||
5690 | - jae 9f | ||
5691 | - | ||
5692 | -10: xorl %eax, %eax | ||
5693 | - retq | ||
5694 | - | ||
5695 | - /* Wake up all threads. */ | ||
5696 | -81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi | ||
5697 | - movl $SYS_futex, %eax | ||
5698 | - movl $1, %edx | ||
5699 | - movl $0x7fffffff, %r10d | ||
5700 | - syscall | ||
5701 | - | ||
5702 | - /* For any kind of error, which mainly is EAGAIN, we try again | ||
5703 | - with WAKE. The general test also covers running on old | ||
5704 | - kernels. */ | ||
5705 | - cmpq $-4095, %rax | ||
5706 | - jb 10b | ||
5707 | - jmp 9f | ||
5708 | - | ||
5709 | - .align 16 | ||
5710 | - /* Unlock. */ | ||
5711 | -4: LOCK | ||
5712 | - decl cond_lock-cond_futex(%rdi) | ||
5713 | - jne 5f | ||
5714 | - | ||
5715 | -6: xorl %eax, %eax | ||
5716 | - retq | ||
5717 | - | ||
5718 | - /* Initial locking failed. */ | ||
5719 | -1: | ||
5720 | -#if cond_lock != 0 | ||
5721 | - addq $cond_lock, %rdi | ||
5722 | -#endif | ||
5723 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
5724 | - movl $LLL_PRIVATE, %eax | ||
5725 | - movl $LLL_SHARED, %esi | ||
5726 | - cmovne %eax, %esi | ||
5727 | - callq __lll_lock_wait | ||
5728 | -#if cond_lock != 0 | ||
5729 | - subq $cond_lock, %rdi | ||
5730 | -#endif | ||
5731 | - jmp 2b | ||
5732 | - | ||
5733 | - /* Unlock in loop requires wakeup. */ | ||
5734 | -5: addq $cond_lock-cond_futex, %rdi | ||
5735 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
5736 | - movl $LLL_PRIVATE, %eax | ||
5737 | - movl $LLL_SHARED, %esi | ||
5738 | - cmovne %eax, %esi | ||
5739 | - callq __lll_unlock_wake | ||
5740 | - jmp 6b | ||
5741 | - | ||
5742 | - /* Unlock in loop requires wakeup. */ | ||
5743 | -7: addq $cond_lock-cond_futex, %rdi | ||
5744 | - cmp $-1, %R8_LP | ||
5745 | - movl $LLL_PRIVATE, %eax | ||
5746 | - movl $LLL_SHARED, %esi | ||
5747 | - cmovne %eax, %esi | ||
5748 | - callq __lll_unlock_wake | ||
5749 | - subq $cond_lock-cond_futex, %rdi | ||
5750 | - jmp 8b | ||
5751 | - | ||
5752 | -9: /* The futex requeue functionality is not available. */ | ||
5753 | - cmp $-1, %R8_LP | ||
5754 | - movl $0x7fffffff, %edx | ||
5755 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5756 | - movl $FUTEX_WAKE, %eax | ||
5757 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
5758 | - cmove %eax, %esi | ||
5759 | -#else | ||
5760 | - movl $0, %eax | ||
5761 | - movl %fs:PRIVATE_FUTEX, %esi | ||
5762 | - cmove %eax, %esi | ||
5763 | - orl $FUTEX_WAKE, %esi | ||
5764 | -#endif | ||
5765 | - movl $SYS_futex, %eax | ||
5766 | - syscall | ||
5767 | - jmp 10b | ||
5768 | -END(__pthread_cond_broadcast) | ||
5769 | - | ||
5770 | -versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast, | ||
5771 | - GLIBC_2_3_2) | ||
5772 | diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S | ||
5773 | deleted file mode 100644 | ||
5774 | index da14bc3..0000000 | ||
5775 | --- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S | ||
5776 | +++ /dev/null | ||
5777 | @@ -1,161 +0,0 @@ | ||
5778 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
5779 | - This file is part of the GNU C Library. | ||
5780 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
5781 | - | ||
5782 | - The GNU C Library is free software; you can redistribute it and/or | ||
5783 | - modify it under the terms of the GNU Lesser General Public | ||
5784 | - License as published by the Free Software Foundation; either | ||
5785 | - version 2.1 of the License, or (at your option) any later version. | ||
5786 | - | ||
5787 | - The GNU C Library is distributed in the hope that it will be useful, | ||
5788 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
5789 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
5790 | - Lesser General Public License for more details. | ||
5791 | - | ||
5792 | - You should have received a copy of the GNU Lesser General Public | ||
5793 | - License along with the GNU C Library; if not, see | ||
5794 | - <http://www.gnu.org/licenses/>. */ | ||
5795 | - | ||
5796 | -#include <sysdep.h> | ||
5797 | -#include <shlib-compat.h> | ||
5798 | -#include <lowlevellock.h> | ||
5799 | -#include <lowlevelcond.h> | ||
5800 | -#include <pthread-pi-defines.h> | ||
5801 | -#include <kernel-features.h> | ||
5802 | -#include <pthread-errnos.h> | ||
5803 | -#include <stap-probe.h> | ||
5804 | - | ||
5805 | - | ||
5806 | - .text | ||
5807 | - | ||
5808 | -ENTRY(__pthread_cond_signal) | ||
5809 | - | ||
5810 | - LIBC_PROBE (cond_signal, 1, %rdi) | ||
5811 | - | ||
5812 | - /* Get internal lock. */ | ||
5813 | - movq %rdi, %r8 | ||
5814 | - movl $1, %esi | ||
5815 | - xorl %eax, %eax | ||
5816 | - LOCK | ||
5817 | -#if cond_lock == 0 | ||
5818 | - cmpxchgl %esi, (%rdi) | ||
5819 | -#else | ||
5820 | - cmpxchgl %esi, cond_lock(%rdi) | ||
5821 | -#endif | ||
5822 | - jnz 1f | ||
5823 | - | ||
5824 | -2: addq $cond_futex, %rdi | ||
5825 | - movq total_seq(%r8), %rcx | ||
5826 | - cmpq wakeup_seq(%r8), %rcx | ||
5827 | - jbe 4f | ||
5828 | - | ||
5829 | - /* Bump the wakeup number. */ | ||
5830 | - addq $1, wakeup_seq(%r8) | ||
5831 | - addl $1, (%rdi) | ||
5832 | - | ||
5833 | - /* Wake up one thread. */ | ||
5834 | - LP_OP(cmp) $-1, dep_mutex(%r8) | ||
5835 | - movl $FUTEX_WAKE_OP, %esi | ||
5836 | - movl $1, %edx | ||
5837 | - movl $SYS_futex, %eax | ||
5838 | - je 8f | ||
5839 | - | ||
5840 | - /* Get the address of the mutex used. */ | ||
5841 | - mov dep_mutex(%r8), %RCX_LP | ||
5842 | - movl MUTEX_KIND(%rcx), %r11d | ||
5843 | - andl $(ROBUST_BIT|PI_BIT), %r11d | ||
5844 | - cmpl $PI_BIT, %r11d | ||
5845 | - je 9f | ||
5846 | - | ||
5847 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5848 | - movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi | ||
5849 | -#else | ||
5850 | - orl %fs:PRIVATE_FUTEX, %esi | ||
5851 | -#endif | ||
5852 | - | ||
5853 | -8: movl $1, %r10d | ||
5854 | -#if cond_lock != 0 | ||
5855 | - addq $cond_lock, %r8 | ||
5856 | -#endif | ||
5857 | - movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %r9d | ||
5858 | - syscall | ||
5859 | -#if cond_lock != 0 | ||
5860 | - subq $cond_lock, %r8 | ||
5861 | -#endif | ||
5862 | - /* For any kind of error, we try again with WAKE. | ||
5863 | - The general test also covers running on old kernels. */ | ||
5864 | - cmpq $-4095, %rax | ||
5865 | - jae 7f | ||
5866 | - | ||
5867 | - xorl %eax, %eax | ||
5868 | - retq | ||
5869 | - | ||
5870 | - /* Wake up one thread and requeue none in the PI Mutex case. */ | ||
5871 | -9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi | ||
5872 | - movq %rcx, %r8 | ||
5873 | - xorq %r10, %r10 | ||
5874 | - movl (%rdi), %r9d // XXX Can this be right? | ||
5875 | - syscall | ||
5876 | - | ||
5877 | - leaq -cond_futex(%rdi), %r8 | ||
5878 | - | ||
5879 | - /* For any kind of error, we try again with WAKE. | ||
5880 | - The general test also covers running on old kernels. */ | ||
5881 | - cmpq $-4095, %rax | ||
5882 | - jb 4f | ||
5883 | - | ||
5884 | -7: | ||
5885 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
5886 | - andl $FUTEX_PRIVATE_FLAG, %esi | ||
5887 | -#else | ||
5888 | - andl %fs:PRIVATE_FUTEX, %esi | ||
5889 | -#endif | ||
5890 | - orl $FUTEX_WAKE, %esi | ||
5891 | - movl $SYS_futex, %eax | ||
5892 | - /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. | ||
5893 | - movl $1, %edx */ | ||
5894 | - syscall | ||
5895 | - | ||
5896 | - /* Unlock. */ | ||
5897 | -4: LOCK | ||
5898 | -#if cond_lock == 0 | ||
5899 | - decl (%r8) | ||
5900 | -#else | ||
5901 | - decl cond_lock(%r8) | ||
5902 | -#endif | ||
5903 | - jne 5f | ||
5904 | - | ||
5905 | -6: xorl %eax, %eax | ||
5906 | - retq | ||
5907 | - | ||
5908 | - /* Initial locking failed. */ | ||
5909 | -1: | ||
5910 | -#if cond_lock != 0 | ||
5911 | - addq $cond_lock, %rdi | ||
5912 | -#endif | ||
5913 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
5914 | - movl $LLL_PRIVATE, %eax | ||
5915 | - movl $LLL_SHARED, %esi | ||
5916 | - cmovne %eax, %esi | ||
5917 | - callq __lll_lock_wait | ||
5918 | -#if cond_lock != 0 | ||
5919 | - subq $cond_lock, %rdi | ||
5920 | -#endif | ||
5921 | - jmp 2b | ||
5922 | - | ||
5923 | - /* Unlock in loop requires wakeup. */ | ||
5924 | -5: | ||
5925 | - movq %r8, %rdi | ||
5926 | -#if cond_lock != 0 | ||
5927 | - addq $cond_lock, %rdi | ||
5928 | -#endif | ||
5929 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
5930 | - movl $LLL_PRIVATE, %eax | ||
5931 | - movl $LLL_SHARED, %esi | ||
5932 | - cmovne %eax, %esi | ||
5933 | - callq __lll_unlock_wake | ||
5934 | - jmp 6b | ||
5935 | -END(__pthread_cond_signal) | ||
5936 | - | ||
5937 | -versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal, | ||
5938 | - GLIBC_2_3_2) | ||
5939 | diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S | ||
5940 | deleted file mode 100644 | ||
5941 | index 82ffa1a..0000000 | ||
5942 | --- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S | ||
5943 | +++ /dev/null | ||
5944 | @@ -1,623 +0,0 @@ | ||
5945 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
5946 | - This file is part of the GNU C Library. | ||
5947 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
5948 | - | ||
5949 | - The GNU C Library is free software; you can redistribute it and/or | ||
5950 | - modify it under the terms of the GNU Lesser General Public | ||
5951 | - License as published by the Free Software Foundation; either | ||
5952 | - version 2.1 of the License, or (at your option) any later version. | ||
5953 | - | ||
5954 | - The GNU C Library is distributed in the hope that it will be useful, | ||
5955 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
5956 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
5957 | - Lesser General Public License for more details. | ||
5958 | - | ||
5959 | - You should have received a copy of the GNU Lesser General Public | ||
5960 | - License along with the GNU C Library; if not, see | ||
5961 | - <http://www.gnu.org/licenses/>. */ | ||
5962 | - | ||
5963 | -#include <sysdep.h> | ||
5964 | -#include <shlib-compat.h> | ||
5965 | -#include <lowlevellock.h> | ||
5966 | -#include <lowlevelcond.h> | ||
5967 | -#include <pthread-pi-defines.h> | ||
5968 | -#include <pthread-errnos.h> | ||
5969 | -#include <stap-probe.h> | ||
5970 | - | ||
5971 | -#include <kernel-features.h> | ||
5972 | - | ||
5973 | - | ||
5974 | - .text | ||
5975 | - | ||
5976 | - | ||
5977 | -/* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, | ||
5978 | - const struct timespec *abstime) */ | ||
5979 | - .globl __pthread_cond_timedwait | ||
5980 | - .type __pthread_cond_timedwait, @function | ||
5981 | - .align 16 | ||
5982 | -__pthread_cond_timedwait: | ||
5983 | -.LSTARTCODE: | ||
5984 | - cfi_startproc | ||
5985 | -#ifdef SHARED | ||
5986 | - cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, | ||
5987 | - DW.ref.__gcc_personality_v0) | ||
5988 | - cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) | ||
5989 | -#else | ||
5990 | - cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) | ||
5991 | - cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) | ||
5992 | -#endif | ||
5993 | - | ||
5994 | - pushq %r12 | ||
5995 | - cfi_adjust_cfa_offset(8) | ||
5996 | - cfi_rel_offset(%r12, 0) | ||
5997 | - pushq %r13 | ||
5998 | - cfi_adjust_cfa_offset(8) | ||
5999 | - cfi_rel_offset(%r13, 0) | ||
6000 | - pushq %r14 | ||
6001 | - cfi_adjust_cfa_offset(8) | ||
6002 | - cfi_rel_offset(%r14, 0) | ||
6003 | - pushq %r15 | ||
6004 | - cfi_adjust_cfa_offset(8) | ||
6005 | - cfi_rel_offset(%r15, 0) | ||
6006 | -#define FRAME_SIZE (32+8) | ||
6007 | - subq $FRAME_SIZE, %rsp | ||
6008 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
6009 | - cfi_remember_state | ||
6010 | - | ||
6011 | - LIBC_PROBE (cond_timedwait, 3, %rdi, %rsi, %rdx) | ||
6012 | - | ||
6013 | - cmpq $1000000000, 8(%rdx) | ||
6014 | - movl $EINVAL, %eax | ||
6015 | - jae 48f | ||
6016 | - | ||
6017 | - /* Stack frame: | ||
6018 | - | ||
6019 | - rsp + 48 | ||
6020 | - +--------------------------+ | ||
6021 | - rsp + 32 | timeout value | | ||
6022 | - +--------------------------+ | ||
6023 | - rsp + 24 | old wake_seq value | | ||
6024 | - +--------------------------+ | ||
6025 | - rsp + 16 | mutex pointer | | ||
6026 | - +--------------------------+ | ||
6027 | - rsp + 8 | condvar pointer | | ||
6028 | - +--------------------------+ | ||
6029 | - rsp + 4 | old broadcast_seq value | | ||
6030 | - +--------------------------+ | ||
6031 | - rsp + 0 | old cancellation mode | | ||
6032 | - +--------------------------+ | ||
6033 | - */ | ||
6034 | - | ||
6035 | - LP_OP(cmp) $-1, dep_mutex(%rdi) | ||
6036 | - | ||
6037 | - /* Prepare structure passed to cancellation handler. */ | ||
6038 | - movq %rdi, 8(%rsp) | ||
6039 | - movq %rsi, 16(%rsp) | ||
6040 | - movq %rdx, %r13 | ||
6041 | - | ||
6042 | - je 22f | ||
6043 | - mov %RSI_LP, dep_mutex(%rdi) | ||
6044 | - | ||
6045 | -22: | ||
6046 | - xorb %r15b, %r15b | ||
6047 | - | ||
6048 | - /* Get internal lock. */ | ||
6049 | - movl $1, %esi | ||
6050 | - xorl %eax, %eax | ||
6051 | - LOCK | ||
6052 | -#if cond_lock == 0 | ||
6053 | - cmpxchgl %esi, (%rdi) | ||
6054 | -#else | ||
6055 | - cmpxchgl %esi, cond_lock(%rdi) | ||
6056 | -#endif | ||
6057 | - jnz 31f | ||
6058 | - | ||
6059 | - /* Unlock the mutex. */ | ||
6060 | -32: movq 16(%rsp), %rdi | ||
6061 | - xorl %esi, %esi | ||
6062 | - callq __pthread_mutex_unlock_usercnt | ||
6063 | - | ||
6064 | - testl %eax, %eax | ||
6065 | - jne 46f | ||
6066 | - | ||
6067 | - movq 8(%rsp), %rdi | ||
6068 | - incq total_seq(%rdi) | ||
6069 | - incl cond_futex(%rdi) | ||
6070 | - addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) | ||
6071 | - | ||
6072 | - /* Get and store current wakeup_seq value. */ | ||
6073 | - movq 8(%rsp), %rdi | ||
6074 | - movq wakeup_seq(%rdi), %r9 | ||
6075 | - movl broadcast_seq(%rdi), %edx | ||
6076 | - movq %r9, 24(%rsp) | ||
6077 | - movl %edx, 4(%rsp) | ||
6078 | - | ||
6079 | - cmpq $0, (%r13) | ||
6080 | - movq $-ETIMEDOUT, %r14 | ||
6081 | - js 36f | ||
6082 | - | ||
6083 | -38: movl cond_futex(%rdi), %r12d | ||
6084 | - | ||
6085 | - /* Unlock. */ | ||
6086 | - LOCK | ||
6087 | -#if cond_lock == 0 | ||
6088 | - decl (%rdi) | ||
6089 | -#else | ||
6090 | - decl cond_lock(%rdi) | ||
6091 | -#endif | ||
6092 | - jne 33f | ||
6093 | - | ||
6094 | -.LcleanupSTART1: | ||
6095 | -34: callq __pthread_enable_asynccancel | ||
6096 | - movl %eax, (%rsp) | ||
6097 | - | ||
6098 | - movq %r13, %r10 | ||
6099 | - movl $FUTEX_WAIT_BITSET, %esi | ||
6100 | - LP_OP(cmp) $-1, dep_mutex(%rdi) | ||
6101 | - je 60f | ||
6102 | - | ||
6103 | - mov dep_mutex(%rdi), %R8_LP | ||
6104 | - /* Requeue to a non-robust PI mutex if the PI bit is set and | ||
6105 | - the robust bit is not set. */ | ||
6106 | - movl MUTEX_KIND(%r8), %eax | ||
6107 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
6108 | - cmpl $PI_BIT, %eax | ||
6109 | - jne 61f | ||
6110 | - | ||
6111 | - movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi | ||
6112 | - xorl %eax, %eax | ||
6113 | - /* The following only works like this because we only support | ||
6114 | - two clocks, represented using a single bit. */ | ||
6115 | - testl $1, cond_nwaiters(%rdi) | ||
6116 | - movl $FUTEX_CLOCK_REALTIME, %edx | ||
6117 | - cmove %edx, %eax | ||
6118 | - orl %eax, %esi | ||
6119 | - movq %r12, %rdx | ||
6120 | - addq $cond_futex, %rdi | ||
6121 | - movl $SYS_futex, %eax | ||
6122 | - syscall | ||
6123 | - | ||
6124 | - cmpl $0, %eax | ||
6125 | - sete %r15b | ||
6126 | - | ||
6127 | -#ifdef __ASSUME_REQUEUE_PI | ||
6128 | - jmp 62f | ||
6129 | -#else | ||
6130 | - je 62f | ||
6131 | - | ||
6132 | - /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns | ||
6133 | - successfully, it has already locked the mutex for us and the | ||
6134 | - pi_flag (%r15b) is set to denote that fact. However, if another | ||
6135 | - thread changed the futex value before we entered the wait, the | ||
6136 | - syscall may return an EAGAIN and the mutex is not locked. We go | ||
6137 | - ahead with a success anyway since later we look at the pi_flag to | ||
6138 | - decide if we got the mutex or not. The sequence numbers then make | ||
6139 | - sure that only one of the threads actually wake up. We retry using | ||
6140 | - normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal | ||
6141 | - and PI futexes don't mix. | ||
6142 | - | ||
6143 | - Note that we don't check for EAGAIN specifically; we assume that the | ||
6144 | - only other error the futex function could return is EAGAIN (barring | ||
6145 | - the ETIMEOUT of course, for the timeout case in futex) since | ||
6146 | - anything else would mean an error in our function. It is too | ||
6147 | - expensive to do that check for every call (which is quite common in | ||
6148 | - case of a large number of threads), so it has been skipped. */ | ||
6149 | - cmpl $-ENOSYS, %eax | ||
6150 | - jne 62f | ||
6151 | - | ||
6152 | - subq $cond_futex, %rdi | ||
6153 | -#endif | ||
6154 | - | ||
6155 | -61: movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi | ||
6156 | -60: xorb %r15b, %r15b | ||
6157 | - xorl %eax, %eax | ||
6158 | - /* The following only works like this because we only support | ||
6159 | - two clocks, represented using a single bit. */ | ||
6160 | - testl $1, cond_nwaiters(%rdi) | ||
6161 | - movl $FUTEX_CLOCK_REALTIME, %edx | ||
6162 | - movl $0xffffffff, %r9d | ||
6163 | - cmove %edx, %eax | ||
6164 | - orl %eax, %esi | ||
6165 | - movq %r12, %rdx | ||
6166 | - addq $cond_futex, %rdi | ||
6167 | - movl $SYS_futex, %eax | ||
6168 | - syscall | ||
6169 | -62: movq %rax, %r14 | ||
6170 | - | ||
6171 | - movl (%rsp), %edi | ||
6172 | - callq __pthread_disable_asynccancel | ||
6173 | -.LcleanupEND1: | ||
6174 | - | ||
6175 | - /* Lock. */ | ||
6176 | - movq 8(%rsp), %rdi | ||
6177 | - movl $1, %esi | ||
6178 | - xorl %eax, %eax | ||
6179 | - LOCK | ||
6180 | -#if cond_lock == 0 | ||
6181 | - cmpxchgl %esi, (%rdi) | ||
6182 | -#else | ||
6183 | - cmpxchgl %esi, cond_lock(%rdi) | ||
6184 | -#endif | ||
6185 | - jne 35f | ||
6186 | - | ||
6187 | -36: movl broadcast_seq(%rdi), %edx | ||
6188 | - | ||
6189 | - movq woken_seq(%rdi), %rax | ||
6190 | - | ||
6191 | - movq wakeup_seq(%rdi), %r9 | ||
6192 | - | ||
6193 | - cmpl 4(%rsp), %edx | ||
6194 | - jne 53f | ||
6195 | - | ||
6196 | - cmpq 24(%rsp), %r9 | ||
6197 | - jbe 45f | ||
6198 | - | ||
6199 | - cmpq %rax, %r9 | ||
6200 | - ja 39f | ||
6201 | - | ||
6202 | -45: cmpq $-ETIMEDOUT, %r14 | ||
6203 | - je 99f | ||
6204 | - | ||
6205 | - /* We need to go back to futex_wait. If we're using requeue_pi, then | ||
6206 | - release the mutex we had acquired and go back. */ | ||
6207 | - test %r15b, %r15b | ||
6208 | - jz 38b | ||
6209 | - | ||
6210 | - /* Adjust the mutex values first and then unlock it. The unlock | ||
6211 | - should always succeed or else the kernel did not lock the | ||
6212 | - mutex correctly. */ | ||
6213 | - movq %r8, %rdi | ||
6214 | - callq __pthread_mutex_cond_lock_adjust | ||
6215 | - xorl %esi, %esi | ||
6216 | - callq __pthread_mutex_unlock_usercnt | ||
6217 | - /* Reload cond_var. */ | ||
6218 | - movq 8(%rsp), %rdi | ||
6219 | - jmp 38b | ||
6220 | - | ||
6221 | -99: incq wakeup_seq(%rdi) | ||
6222 | - incl cond_futex(%rdi) | ||
6223 | - movl $ETIMEDOUT, %r14d | ||
6224 | - jmp 44f | ||
6225 | - | ||
6226 | -53: xorq %r14, %r14 | ||
6227 | - jmp 54f | ||
6228 | - | ||
6229 | -39: xorq %r14, %r14 | ||
6230 | -44: incq woken_seq(%rdi) | ||
6231 | - | ||
6232 | -54: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) | ||
6233 | - | ||
6234 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
6235 | - cmpq $0xffffffffffffffff, total_seq(%rdi) | ||
6236 | - jne 55f | ||
6237 | - movl cond_nwaiters(%rdi), %eax | ||
6238 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
6239 | - jne 55f | ||
6240 | - | ||
6241 | - addq $cond_nwaiters, %rdi | ||
6242 | - LP_OP(cmp) $-1, dep_mutex-cond_nwaiters(%rdi) | ||
6243 | - movl $1, %edx | ||
6244 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
6245 | - movl $FUTEX_WAKE, %eax | ||
6246 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
6247 | - cmove %eax, %esi | ||
6248 | -#else | ||
6249 | - movl $0, %eax | ||
6250 | - movl %fs:PRIVATE_FUTEX, %esi | ||
6251 | - cmove %eax, %esi | ||
6252 | - orl $FUTEX_WAKE, %esi | ||
6253 | -#endif | ||
6254 | - movl $SYS_futex, %eax | ||
6255 | - syscall | ||
6256 | - subq $cond_nwaiters, %rdi | ||
6257 | - | ||
6258 | -55: LOCK | ||
6259 | -#if cond_lock == 0 | ||
6260 | - decl (%rdi) | ||
6261 | -#else | ||
6262 | - decl cond_lock(%rdi) | ||
6263 | -#endif | ||
6264 | - jne 40f | ||
6265 | - | ||
6266 | - /* If requeue_pi is used the kernel performs the locking of the | ||
6267 | - mutex. */ | ||
6268 | -41: movq 16(%rsp), %rdi | ||
6269 | - testb %r15b, %r15b | ||
6270 | - jnz 64f | ||
6271 | - | ||
6272 | - callq __pthread_mutex_cond_lock | ||
6273 | - | ||
6274 | -63: testq %rax, %rax | ||
6275 | - cmoveq %r14, %rax | ||
6276 | - | ||
6277 | -48: addq $FRAME_SIZE, %rsp | ||
6278 | - cfi_adjust_cfa_offset(-FRAME_SIZE) | ||
6279 | - popq %r15 | ||
6280 | - cfi_adjust_cfa_offset(-8) | ||
6281 | - cfi_restore(%r15) | ||
6282 | - popq %r14 | ||
6283 | - cfi_adjust_cfa_offset(-8) | ||
6284 | - cfi_restore(%r14) | ||
6285 | - popq %r13 | ||
6286 | - cfi_adjust_cfa_offset(-8) | ||
6287 | - cfi_restore(%r13) | ||
6288 | - popq %r12 | ||
6289 | - cfi_adjust_cfa_offset(-8) | ||
6290 | - cfi_restore(%r12) | ||
6291 | - | ||
6292 | - retq | ||
6293 | - | ||
6294 | - cfi_restore_state | ||
6295 | - | ||
6296 | -64: callq __pthread_mutex_cond_lock_adjust | ||
6297 | - movq %r14, %rax | ||
6298 | - jmp 48b | ||
6299 | - | ||
6300 | - /* Initial locking failed. */ | ||
6301 | -31: | ||
6302 | -#if cond_lock != 0 | ||
6303 | - addq $cond_lock, %rdi | ||
6304 | -#endif | ||
6305 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6306 | - movl $LLL_PRIVATE, %eax | ||
6307 | - movl $LLL_SHARED, %esi | ||
6308 | - cmovne %eax, %esi | ||
6309 | - callq __lll_lock_wait | ||
6310 | - jmp 32b | ||
6311 | - | ||
6312 | - /* Unlock in loop requires wakeup. */ | ||
6313 | -33: | ||
6314 | -#if cond_lock != 0 | ||
6315 | - addq $cond_lock, %rdi | ||
6316 | -#endif | ||
6317 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6318 | - movl $LLL_PRIVATE, %eax | ||
6319 | - movl $LLL_SHARED, %esi | ||
6320 | - cmovne %eax, %esi | ||
6321 | - callq __lll_unlock_wake | ||
6322 | - jmp 34b | ||
6323 | - | ||
6324 | - /* Locking in loop failed. */ | ||
6325 | -35: | ||
6326 | -#if cond_lock != 0 | ||
6327 | - addq $cond_lock, %rdi | ||
6328 | -#endif | ||
6329 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6330 | - movl $LLL_PRIVATE, %eax | ||
6331 | - movl $LLL_SHARED, %esi | ||
6332 | - cmovne %eax, %esi | ||
6333 | - callq __lll_lock_wait | ||
6334 | -#if cond_lock != 0 | ||
6335 | - subq $cond_lock, %rdi | ||
6336 | -#endif | ||
6337 | - jmp 36b | ||
6338 | - | ||
6339 | - /* Unlock after loop requires wakeup. */ | ||
6340 | -40: | ||
6341 | -#if cond_lock != 0 | ||
6342 | - addq $cond_lock, %rdi | ||
6343 | -#endif | ||
6344 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6345 | - movl $LLL_PRIVATE, %eax | ||
6346 | - movl $LLL_SHARED, %esi | ||
6347 | - cmovne %eax, %esi | ||
6348 | - callq __lll_unlock_wake | ||
6349 | - jmp 41b | ||
6350 | - | ||
6351 | - /* The initial unlocking of the mutex failed. */ | ||
6352 | -46: movq 8(%rsp), %rdi | ||
6353 | - movq %rax, (%rsp) | ||
6354 | - LOCK | ||
6355 | -#if cond_lock == 0 | ||
6356 | - decl (%rdi) | ||
6357 | -#else | ||
6358 | - decl cond_lock(%rdi) | ||
6359 | -#endif | ||
6360 | - jne 47f | ||
6361 | - | ||
6362 | -#if cond_lock != 0 | ||
6363 | - addq $cond_lock, %rdi | ||
6364 | -#endif | ||
6365 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6366 | - movl $LLL_PRIVATE, %eax | ||
6367 | - movl $LLL_SHARED, %esi | ||
6368 | - cmovne %eax, %esi | ||
6369 | - callq __lll_unlock_wake | ||
6370 | - | ||
6371 | -47: movq (%rsp), %rax | ||
6372 | - jmp 48b | ||
6373 | - | ||
6374 | - .size __pthread_cond_timedwait, .-__pthread_cond_timedwait | ||
6375 | -versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait, | ||
6376 | - GLIBC_2_3_2) | ||
6377 | - | ||
6378 | - | ||
6379 | - .align 16 | ||
6380 | - .type __condvar_cleanup2, @function | ||
6381 | -__condvar_cleanup2: | ||
6382 | - /* Stack frame: | ||
6383 | - | ||
6384 | - rsp + 72 | ||
6385 | - +--------------------------+ | ||
6386 | - rsp + 64 | %r12 | | ||
6387 | - +--------------------------+ | ||
6388 | - rsp + 56 | %r13 | | ||
6389 | - +--------------------------+ | ||
6390 | - rsp + 48 | %r14 | | ||
6391 | - +--------------------------+ | ||
6392 | - rsp + 24 | unused | | ||
6393 | - +--------------------------+ | ||
6394 | - rsp + 16 | mutex pointer | | ||
6395 | - +--------------------------+ | ||
6396 | - rsp + 8 | condvar pointer | | ||
6397 | - +--------------------------+ | ||
6398 | - rsp + 4 | old broadcast_seq value | | ||
6399 | - +--------------------------+ | ||
6400 | - rsp + 0 | old cancellation mode | | ||
6401 | - +--------------------------+ | ||
6402 | - */ | ||
6403 | - | ||
6404 | - movq %rax, 24(%rsp) | ||
6405 | - | ||
6406 | - /* Get internal lock. */ | ||
6407 | - movq 8(%rsp), %rdi | ||
6408 | - movl $1, %esi | ||
6409 | - xorl %eax, %eax | ||
6410 | - LOCK | ||
6411 | -#if cond_lock == 0 | ||
6412 | - cmpxchgl %esi, (%rdi) | ||
6413 | -#else | ||
6414 | - cmpxchgl %esi, cond_lock(%rdi) | ||
6415 | -#endif | ||
6416 | - jz 1f | ||
6417 | - | ||
6418 | -#if cond_lock != 0 | ||
6419 | - addq $cond_lock, %rdi | ||
6420 | -#endif | ||
6421 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6422 | - movl $LLL_PRIVATE, %eax | ||
6423 | - movl $LLL_SHARED, %esi | ||
6424 | - cmovne %eax, %esi | ||
6425 | - callq __lll_lock_wait | ||
6426 | -#if cond_lock != 0 | ||
6427 | - subq $cond_lock, %rdi | ||
6428 | -#endif | ||
6429 | - | ||
6430 | -1: movl broadcast_seq(%rdi), %edx | ||
6431 | - cmpl 4(%rsp), %edx | ||
6432 | - jne 3f | ||
6433 | - | ||
6434 | - /* We increment the wakeup_seq counter only if it is lower than | ||
6435 | - total_seq. If this is not the case the thread was woken and | ||
6436 | - then canceled. In this case we ignore the signal. */ | ||
6437 | - movq total_seq(%rdi), %rax | ||
6438 | - cmpq wakeup_seq(%rdi), %rax | ||
6439 | - jbe 6f | ||
6440 | - incq wakeup_seq(%rdi) | ||
6441 | - incl cond_futex(%rdi) | ||
6442 | -6: incq woken_seq(%rdi) | ||
6443 | - | ||
6444 | -3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) | ||
6445 | - | ||
6446 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
6447 | - xorq %r12, %r12 | ||
6448 | - cmpq $0xffffffffffffffff, total_seq(%rdi) | ||
6449 | - jne 4f | ||
6450 | - movl cond_nwaiters(%rdi), %eax | ||
6451 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
6452 | - jne 4f | ||
6453 | - | ||
6454 | - LP_OP(cmp) $-1, dep_mutex(%rdi) | ||
6455 | - leaq cond_nwaiters(%rdi), %rdi | ||
6456 | - movl $1, %edx | ||
6457 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
6458 | - movl $FUTEX_WAKE, %eax | ||
6459 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
6460 | - cmove %eax, %esi | ||
6461 | -#else | ||
6462 | - movl $0, %eax | ||
6463 | - movl %fs:PRIVATE_FUTEX, %esi | ||
6464 | - cmove %eax, %esi | ||
6465 | - orl $FUTEX_WAKE, %esi | ||
6466 | -#endif | ||
6467 | - movl $SYS_futex, %eax | ||
6468 | - syscall | ||
6469 | - subq $cond_nwaiters, %rdi | ||
6470 | - movl $1, %r12d | ||
6471 | - | ||
6472 | -4: LOCK | ||
6473 | -#if cond_lock == 0 | ||
6474 | - decl (%rdi) | ||
6475 | -#else | ||
6476 | - decl cond_lock(%rdi) | ||
6477 | -#endif | ||
6478 | - je 2f | ||
6479 | -#if cond_lock != 0 | ||
6480 | - addq $cond_lock, %rdi | ||
6481 | -#endif | ||
6482 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6483 | - movl $LLL_PRIVATE, %eax | ||
6484 | - movl $LLL_SHARED, %esi | ||
6485 | - cmovne %eax, %esi | ||
6486 | - callq __lll_unlock_wake | ||
6487 | - | ||
6488 | - /* Wake up all waiters to make sure no signal gets lost. */ | ||
6489 | -2: testq %r12, %r12 | ||
6490 | - jnz 5f | ||
6491 | - addq $cond_futex, %rdi | ||
6492 | - LP_OP(cmp) $-1, dep_mutex-cond_futex(%rdi) | ||
6493 | - movl $0x7fffffff, %edx | ||
6494 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
6495 | - movl $FUTEX_WAKE, %eax | ||
6496 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
6497 | - cmove %eax, %esi | ||
6498 | -#else | ||
6499 | - movl $0, %eax | ||
6500 | - movl %fs:PRIVATE_FUTEX, %esi | ||
6501 | - cmove %eax, %esi | ||
6502 | - orl $FUTEX_WAKE, %esi | ||
6503 | -#endif | ||
6504 | - movl $SYS_futex, %eax | ||
6505 | - syscall | ||
6506 | - | ||
6507 | - /* Lock the mutex only if we don't own it already. This only happens | ||
6508 | - in case of PI mutexes, if we got cancelled after a successful | ||
6509 | - return of the futex syscall and before disabling async | ||
6510 | - cancellation. */ | ||
6511 | -5: movq 16(%rsp), %rdi | ||
6512 | - movl MUTEX_KIND(%rdi), %eax | ||
6513 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
6514 | - cmpl $PI_BIT, %eax | ||
6515 | - jne 7f | ||
6516 | - | ||
6517 | - movl (%rdi), %eax | ||
6518 | - andl $TID_MASK, %eax | ||
6519 | - cmpl %eax, %fs:TID | ||
6520 | - jne 7f | ||
6521 | - /* We managed to get the lock. Fix it up before returning. */ | ||
6522 | - callq __pthread_mutex_cond_lock_adjust | ||
6523 | - jmp 8f | ||
6524 | - | ||
6525 | -7: callq __pthread_mutex_cond_lock | ||
6526 | - | ||
6527 | -8: movq 24(%rsp), %rdi | ||
6528 | - movq FRAME_SIZE(%rsp), %r15 | ||
6529 | - movq FRAME_SIZE+8(%rsp), %r14 | ||
6530 | - movq FRAME_SIZE+16(%rsp), %r13 | ||
6531 | - movq FRAME_SIZE+24(%rsp), %r12 | ||
6532 | -.LcallUR: | ||
6533 | - call _Unwind_Resume | ||
6534 | - hlt | ||
6535 | -.LENDCODE: | ||
6536 | - cfi_endproc | ||
6537 | - .size __condvar_cleanup2, .-__condvar_cleanup2 | ||
6538 | - | ||
6539 | - | ||
6540 | - .section .gcc_except_table,"a",@progbits | ||
6541 | -.LexceptSTART: | ||
6542 | - .byte DW_EH_PE_omit # @LPStart format | ||
6543 | - .byte DW_EH_PE_omit # @TType format | ||
6544 | - .byte DW_EH_PE_uleb128 # call-site format | ||
6545 | - .uleb128 .Lcstend-.Lcstbegin | ||
6546 | -.Lcstbegin: | ||
6547 | - .uleb128 .LcleanupSTART1-.LSTARTCODE | ||
6548 | - .uleb128 .LcleanupEND1-.LcleanupSTART1 | ||
6549 | - .uleb128 __condvar_cleanup2-.LSTARTCODE | ||
6550 | - .uleb128 0 | ||
6551 | - .uleb128 .LcallUR-.LSTARTCODE | ||
6552 | - .uleb128 .LENDCODE-.LcallUR | ||
6553 | - .uleb128 0 | ||
6554 | - .uleb128 0 | ||
6555 | -.Lcstend: | ||
6556 | - | ||
6557 | - | ||
6558 | -#ifdef SHARED | ||
6559 | - .hidden DW.ref.__gcc_personality_v0 | ||
6560 | - .weak DW.ref.__gcc_personality_v0 | ||
6561 | - .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits | ||
6562 | - .align LP_SIZE | ||
6563 | - .type DW.ref.__gcc_personality_v0, @object | ||
6564 | - .size DW.ref.__gcc_personality_v0, LP_SIZE | ||
6565 | -DW.ref.__gcc_personality_v0: | ||
6566 | - ASM_ADDR __gcc_personality_v0 | ||
6567 | -#endif | ||
6568 | diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | ||
6569 | deleted file mode 100644 | ||
6570 | index c82f37b..0000000 | ||
6571 | --- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | ||
6572 | +++ /dev/null | ||
6573 | @@ -1,555 +0,0 @@ | ||
6574 | -/* Copyright (C) 2002-2016 Free Software Foundation, Inc. | ||
6575 | - This file is part of the GNU C Library. | ||
6576 | - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. | ||
6577 | - | ||
6578 | - The GNU C Library is free software; you can redistribute it and/or | ||
6579 | - modify it under the terms of the GNU Lesser General Public | ||
6580 | - License as published by the Free Software Foundation; either | ||
6581 | - version 2.1 of the License, or (at your option) any later version. | ||
6582 | - | ||
6583 | - The GNU C Library is distributed in the hope that it will be useful, | ||
6584 | - but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
6585 | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
6586 | - Lesser General Public License for more details. | ||
6587 | - | ||
6588 | - You should have received a copy of the GNU Lesser General Public | ||
6589 | - License along with the GNU C Library; if not, see | ||
6590 | - <http://www.gnu.org/licenses/>. */ | ||
6591 | - | ||
6592 | -#include <sysdep.h> | ||
6593 | -#include <shlib-compat.h> | ||
6594 | -#include <lowlevellock.h> | ||
6595 | -#include <lowlevelcond.h> | ||
6596 | -#include <tcb-offsets.h> | ||
6597 | -#include <pthread-pi-defines.h> | ||
6598 | -#include <pthread-errnos.h> | ||
6599 | -#include <stap-probe.h> | ||
6600 | - | ||
6601 | -#include <kernel-features.h> | ||
6602 | - | ||
6603 | - | ||
6604 | - .text | ||
6605 | - | ||
6606 | -/* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */ | ||
6607 | - .globl __pthread_cond_wait | ||
6608 | - .type __pthread_cond_wait, @function | ||
6609 | - .align 16 | ||
6610 | -__pthread_cond_wait: | ||
6611 | -.LSTARTCODE: | ||
6612 | - cfi_startproc | ||
6613 | -#ifdef SHARED | ||
6614 | - cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, | ||
6615 | - DW.ref.__gcc_personality_v0) | ||
6616 | - cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) | ||
6617 | -#else | ||
6618 | - cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) | ||
6619 | - cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) | ||
6620 | -#endif | ||
6621 | - | ||
6622 | -#define FRAME_SIZE (32+8) | ||
6623 | - leaq -FRAME_SIZE(%rsp), %rsp | ||
6624 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
6625 | - | ||
6626 | - /* Stack frame: | ||
6627 | - | ||
6628 | - rsp + 32 | ||
6629 | - +--------------------------+ | ||
6630 | - rsp + 24 | old wake_seq value | | ||
6631 | - +--------------------------+ | ||
6632 | - rsp + 16 | mutex pointer | | ||
6633 | - +--------------------------+ | ||
6634 | - rsp + 8 | condvar pointer | | ||
6635 | - +--------------------------+ | ||
6636 | - rsp + 4 | old broadcast_seq value | | ||
6637 | - +--------------------------+ | ||
6638 | - rsp + 0 | old cancellation mode | | ||
6639 | - +--------------------------+ | ||
6640 | - */ | ||
6641 | - | ||
6642 | - LIBC_PROBE (cond_wait, 2, %rdi, %rsi) | ||
6643 | - | ||
6644 | - LP_OP(cmp) $-1, dep_mutex(%rdi) | ||
6645 | - | ||
6646 | - /* Prepare structure passed to cancellation handler. */ | ||
6647 | - movq %rdi, 8(%rsp) | ||
6648 | - movq %rsi, 16(%rsp) | ||
6649 | - | ||
6650 | - je 15f | ||
6651 | - mov %RSI_LP, dep_mutex(%rdi) | ||
6652 | - | ||
6653 | - /* Get internal lock. */ | ||
6654 | -15: movl $1, %esi | ||
6655 | - xorl %eax, %eax | ||
6656 | - LOCK | ||
6657 | -#if cond_lock == 0 | ||
6658 | - cmpxchgl %esi, (%rdi) | ||
6659 | -#else | ||
6660 | - cmpxchgl %esi, cond_lock(%rdi) | ||
6661 | -#endif | ||
6662 | - jne 1f | ||
6663 | - | ||
6664 | - /* Unlock the mutex. */ | ||
6665 | -2: movq 16(%rsp), %rdi | ||
6666 | - xorl %esi, %esi | ||
6667 | - callq __pthread_mutex_unlock_usercnt | ||
6668 | - | ||
6669 | - testl %eax, %eax | ||
6670 | - jne 12f | ||
6671 | - | ||
6672 | - movq 8(%rsp), %rdi | ||
6673 | - incq total_seq(%rdi) | ||
6674 | - incl cond_futex(%rdi) | ||
6675 | - addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) | ||
6676 | - | ||
6677 | - /* Get and store current wakeup_seq value. */ | ||
6678 | - movq 8(%rsp), %rdi | ||
6679 | - movq wakeup_seq(%rdi), %r9 | ||
6680 | - movl broadcast_seq(%rdi), %edx | ||
6681 | - movq %r9, 24(%rsp) | ||
6682 | - movl %edx, 4(%rsp) | ||
6683 | - | ||
6684 | - /* Unlock. */ | ||
6685 | -8: movl cond_futex(%rdi), %edx | ||
6686 | - LOCK | ||
6687 | -#if cond_lock == 0 | ||
6688 | - decl (%rdi) | ||
6689 | -#else | ||
6690 | - decl cond_lock(%rdi) | ||
6691 | -#endif | ||
6692 | - jne 3f | ||
6693 | - | ||
6694 | -.LcleanupSTART: | ||
6695 | -4: callq __pthread_enable_asynccancel | ||
6696 | - movl %eax, (%rsp) | ||
6697 | - | ||
6698 | - xorq %r10, %r10 | ||
6699 | - LP_OP(cmp) $-1, dep_mutex(%rdi) | ||
6700 | - leaq cond_futex(%rdi), %rdi | ||
6701 | - movl $FUTEX_WAIT, %esi | ||
6702 | - je 60f | ||
6703 | - | ||
6704 | - mov dep_mutex-cond_futex(%rdi), %R8_LP | ||
6705 | - /* Requeue to a non-robust PI mutex if the PI bit is set and | ||
6706 | - the robust bit is not set. */ | ||
6707 | - movl MUTEX_KIND(%r8), %eax | ||
6708 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
6709 | - cmpl $PI_BIT, %eax | ||
6710 | - jne 61f | ||
6711 | - | ||
6712 | - movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi | ||
6713 | - movl $SYS_futex, %eax | ||
6714 | - syscall | ||
6715 | - | ||
6716 | - cmpl $0, %eax | ||
6717 | - sete %r8b | ||
6718 | - | ||
6719 | -#ifdef __ASSUME_REQUEUE_PI | ||
6720 | - jmp 62f | ||
6721 | -#else | ||
6722 | - je 62f | ||
6723 | - | ||
6724 | - /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns | ||
6725 | - successfully, it has already locked the mutex for us and the | ||
6726 | - pi_flag (%r8b) is set to denote that fact. However, if another | ||
6727 | - thread changed the futex value before we entered the wait, the | ||
6728 | - syscall may return an EAGAIN and the mutex is not locked. We go | ||
6729 | - ahead with a success anyway since later we look at the pi_flag to | ||
6730 | - decide if we got the mutex or not. The sequence numbers then make | ||
6731 | - sure that only one of the threads actually wake up. We retry using | ||
6732 | - normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal | ||
6733 | - and PI futexes don't mix. | ||
6734 | - | ||
6735 | - Note that we don't check for EAGAIN specifically; we assume that the | ||
6736 | - only other error the futex function could return is EAGAIN since | ||
6737 | - anything else would mean an error in our function. It is too | ||
6738 | - expensive to do that check for every call (which is quite common in | ||
6739 | - case of a large number of threads), so it has been skipped. */ | ||
6740 | - cmpl $-ENOSYS, %eax | ||
6741 | - jne 62f | ||
6742 | - | ||
6743 | -# ifndef __ASSUME_PRIVATE_FUTEX | ||
6744 | - movl $FUTEX_WAIT, %esi | ||
6745 | -# endif | ||
6746 | -#endif | ||
6747 | - | ||
6748 | -61: | ||
6749 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
6750 | - movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi | ||
6751 | -#else | ||
6752 | - orl %fs:PRIVATE_FUTEX, %esi | ||
6753 | -#endif | ||
6754 | -60: xorb %r8b, %r8b | ||
6755 | - movl $SYS_futex, %eax | ||
6756 | - syscall | ||
6757 | - | ||
6758 | -62: movl (%rsp), %edi | ||
6759 | - callq __pthread_disable_asynccancel | ||
6760 | -.LcleanupEND: | ||
6761 | - | ||
6762 | - /* Lock. */ | ||
6763 | - movq 8(%rsp), %rdi | ||
6764 | - movl $1, %esi | ||
6765 | - xorl %eax, %eax | ||
6766 | - LOCK | ||
6767 | -#if cond_lock == 0 | ||
6768 | - cmpxchgl %esi, (%rdi) | ||
6769 | -#else | ||
6770 | - cmpxchgl %esi, cond_lock(%rdi) | ||
6771 | -#endif | ||
6772 | - jnz 5f | ||
6773 | - | ||
6774 | -6: movl broadcast_seq(%rdi), %edx | ||
6775 | - | ||
6776 | - movq woken_seq(%rdi), %rax | ||
6777 | - | ||
6778 | - movq wakeup_seq(%rdi), %r9 | ||
6779 | - | ||
6780 | - cmpl 4(%rsp), %edx | ||
6781 | - jne 16f | ||
6782 | - | ||
6783 | - cmpq 24(%rsp), %r9 | ||
6784 | - jbe 19f | ||
6785 | - | ||
6786 | - cmpq %rax, %r9 | ||
6787 | - jna 19f | ||
6788 | - | ||
6789 | - incq woken_seq(%rdi) | ||
6790 | - | ||
6791 | - /* Unlock */ | ||
6792 | -16: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) | ||
6793 | - | ||
6794 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
6795 | - cmpq $0xffffffffffffffff, total_seq(%rdi) | ||
6796 | - jne 17f | ||
6797 | - movl cond_nwaiters(%rdi), %eax | ||
6798 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
6799 | - jne 17f | ||
6800 | - | ||
6801 | - addq $cond_nwaiters, %rdi | ||
6802 | - LP_OP(cmp) $-1, dep_mutex-cond_nwaiters(%rdi) | ||
6803 | - movl $1, %edx | ||
6804 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
6805 | - movl $FUTEX_WAKE, %eax | ||
6806 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
6807 | - cmove %eax, %esi | ||
6808 | -#else | ||
6809 | - movl $0, %eax | ||
6810 | - movl %fs:PRIVATE_FUTEX, %esi | ||
6811 | - cmove %eax, %esi | ||
6812 | - orl $FUTEX_WAKE, %esi | ||
6813 | -#endif | ||
6814 | - movl $SYS_futex, %eax | ||
6815 | - syscall | ||
6816 | - subq $cond_nwaiters, %rdi | ||
6817 | - | ||
6818 | -17: LOCK | ||
6819 | -#if cond_lock == 0 | ||
6820 | - decl (%rdi) | ||
6821 | -#else | ||
6822 | - decl cond_lock(%rdi) | ||
6823 | -#endif | ||
6824 | - jne 10f | ||
6825 | - | ||
6826 | - /* If requeue_pi is used the kernel performs the locking of the | ||
6827 | - mutex. */ | ||
6828 | -11: movq 16(%rsp), %rdi | ||
6829 | - testb %r8b, %r8b | ||
6830 | - jnz 18f | ||
6831 | - | ||
6832 | - callq __pthread_mutex_cond_lock | ||
6833 | - | ||
6834 | -14: leaq FRAME_SIZE(%rsp), %rsp | ||
6835 | - cfi_adjust_cfa_offset(-FRAME_SIZE) | ||
6836 | - | ||
6837 | - /* We return the result of the mutex_lock operation. */ | ||
6838 | - retq | ||
6839 | - | ||
6840 | - cfi_adjust_cfa_offset(FRAME_SIZE) | ||
6841 | - | ||
6842 | -18: callq __pthread_mutex_cond_lock_adjust | ||
6843 | - xorl %eax, %eax | ||
6844 | - jmp 14b | ||
6845 | - | ||
6846 | - /* We need to go back to futex_wait. If we're using requeue_pi, then | ||
6847 | - release the mutex we had acquired and go back. */ | ||
6848 | -19: testb %r8b, %r8b | ||
6849 | - jz 8b | ||
6850 | - | ||
6851 | - /* Adjust the mutex values first and then unlock it. The unlock | ||
6852 | - should always succeed or else the kernel did not lock the mutex | ||
6853 | - correctly. */ | ||
6854 | - movq 16(%rsp), %rdi | ||
6855 | - callq __pthread_mutex_cond_lock_adjust | ||
6856 | - movq %rdi, %r8 | ||
6857 | - xorl %esi, %esi | ||
6858 | - callq __pthread_mutex_unlock_usercnt | ||
6859 | - /* Reload cond_var. */ | ||
6860 | - movq 8(%rsp), %rdi | ||
6861 | - jmp 8b | ||
6862 | - | ||
6863 | - /* Initial locking failed. */ | ||
6864 | -1: | ||
6865 | -#if cond_lock != 0 | ||
6866 | - addq $cond_lock, %rdi | ||
6867 | -#endif | ||
6868 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6869 | - movl $LLL_PRIVATE, %eax | ||
6870 | - movl $LLL_SHARED, %esi | ||
6871 | - cmovne %eax, %esi | ||
6872 | - callq __lll_lock_wait | ||
6873 | - jmp 2b | ||
6874 | - | ||
6875 | - /* Unlock in loop requires wakeup. */ | ||
6876 | -3: | ||
6877 | -#if cond_lock != 0 | ||
6878 | - addq $cond_lock, %rdi | ||
6879 | -#endif | ||
6880 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6881 | - movl $LLL_PRIVATE, %eax | ||
6882 | - movl $LLL_SHARED, %esi | ||
6883 | - cmovne %eax, %esi | ||
6884 | - /* The call preserves %rdx. */ | ||
6885 | - callq __lll_unlock_wake | ||
6886 | -#if cond_lock != 0 | ||
6887 | - subq $cond_lock, %rdi | ||
6888 | -#endif | ||
6889 | - jmp 4b | ||
6890 | - | ||
6891 | - /* Locking in loop failed. */ | ||
6892 | -5: | ||
6893 | -#if cond_lock != 0 | ||
6894 | - addq $cond_lock, %rdi | ||
6895 | -#endif | ||
6896 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6897 | - movl $LLL_PRIVATE, %eax | ||
6898 | - movl $LLL_SHARED, %esi | ||
6899 | - cmovne %eax, %esi | ||
6900 | - callq __lll_lock_wait | ||
6901 | -#if cond_lock != 0 | ||
6902 | - subq $cond_lock, %rdi | ||
6903 | -#endif | ||
6904 | - jmp 6b | ||
6905 | - | ||
6906 | - /* Unlock after loop requires wakeup. */ | ||
6907 | -10: | ||
6908 | -#if cond_lock != 0 | ||
6909 | - addq $cond_lock, %rdi | ||
6910 | -#endif | ||
6911 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6912 | - movl $LLL_PRIVATE, %eax | ||
6913 | - movl $LLL_SHARED, %esi | ||
6914 | - cmovne %eax, %esi | ||
6915 | - callq __lll_unlock_wake | ||
6916 | - jmp 11b | ||
6917 | - | ||
6918 | - /* The initial unlocking of the mutex failed. */ | ||
6919 | -12: movq %rax, %r10 | ||
6920 | - movq 8(%rsp), %rdi | ||
6921 | - LOCK | ||
6922 | -#if cond_lock == 0 | ||
6923 | - decl (%rdi) | ||
6924 | -#else | ||
6925 | - decl cond_lock(%rdi) | ||
6926 | -#endif | ||
6927 | - je 13f | ||
6928 | - | ||
6929 | -#if cond_lock != 0 | ||
6930 | - addq $cond_lock, %rdi | ||
6931 | -#endif | ||
6932 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6933 | - movl $LLL_PRIVATE, %eax | ||
6934 | - movl $LLL_SHARED, %esi | ||
6935 | - cmovne %eax, %esi | ||
6936 | - callq __lll_unlock_wake | ||
6937 | - | ||
6938 | -13: movq %r10, %rax | ||
6939 | - jmp 14b | ||
6940 | - | ||
6941 | - .size __pthread_cond_wait, .-__pthread_cond_wait | ||
6942 | -versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait, | ||
6943 | - GLIBC_2_3_2) | ||
6944 | - | ||
6945 | - | ||
6946 | - .align 16 | ||
6947 | - .type __condvar_cleanup1, @function | ||
6948 | - .globl __condvar_cleanup1 | ||
6949 | - .hidden __condvar_cleanup1 | ||
6950 | -__condvar_cleanup1: | ||
6951 | - /* Stack frame: | ||
6952 | - | ||
6953 | - rsp + 32 | ||
6954 | - +--------------------------+ | ||
6955 | - rsp + 24 | unused | | ||
6956 | - +--------------------------+ | ||
6957 | - rsp + 16 | mutex pointer | | ||
6958 | - +--------------------------+ | ||
6959 | - rsp + 8 | condvar pointer | | ||
6960 | - +--------------------------+ | ||
6961 | - rsp + 4 | old broadcast_seq value | | ||
6962 | - +--------------------------+ | ||
6963 | - rsp + 0 | old cancellation mode | | ||
6964 | - +--------------------------+ | ||
6965 | - */ | ||
6966 | - | ||
6967 | - movq %rax, 24(%rsp) | ||
6968 | - | ||
6969 | - /* Get internal lock. */ | ||
6970 | - movq 8(%rsp), %rdi | ||
6971 | - movl $1, %esi | ||
6972 | - xorl %eax, %eax | ||
6973 | - LOCK | ||
6974 | -#if cond_lock == 0 | ||
6975 | - cmpxchgl %esi, (%rdi) | ||
6976 | -#else | ||
6977 | - cmpxchgl %esi, cond_lock(%rdi) | ||
6978 | -#endif | ||
6979 | - jz 1f | ||
6980 | - | ||
6981 | -#if cond_lock != 0 | ||
6982 | - addq $cond_lock, %rdi | ||
6983 | -#endif | ||
6984 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
6985 | - movl $LLL_PRIVATE, %eax | ||
6986 | - movl $LLL_SHARED, %esi | ||
6987 | - cmovne %eax, %esi | ||
6988 | - callq __lll_lock_wait | ||
6989 | -#if cond_lock != 0 | ||
6990 | - subq $cond_lock, %rdi | ||
6991 | -#endif | ||
6992 | - | ||
6993 | -1: movl broadcast_seq(%rdi), %edx | ||
6994 | - cmpl 4(%rsp), %edx | ||
6995 | - jne 3f | ||
6996 | - | ||
6997 | - /* We increment the wakeup_seq counter only if it is lower than | ||
6998 | - total_seq. If this is not the case the thread was woken and | ||
6999 | - then canceled. In this case we ignore the signal. */ | ||
7000 | - movq total_seq(%rdi), %rax | ||
7001 | - cmpq wakeup_seq(%rdi), %rax | ||
7002 | - jbe 6f | ||
7003 | - incq wakeup_seq(%rdi) | ||
7004 | - incl cond_futex(%rdi) | ||
7005 | -6: incq woken_seq(%rdi) | ||
7006 | - | ||
7007 | -3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) | ||
7008 | - | ||
7009 | - /* Wake up a thread which wants to destroy the condvar object. */ | ||
7010 | - xorl %ecx, %ecx | ||
7011 | - cmpq $0xffffffffffffffff, total_seq(%rdi) | ||
7012 | - jne 4f | ||
7013 | - movl cond_nwaiters(%rdi), %eax | ||
7014 | - andl $~((1 << nwaiters_shift) - 1), %eax | ||
7015 | - jne 4f | ||
7016 | - | ||
7017 | - LP_OP(cmp) $-1, dep_mutex(%rdi) | ||
7018 | - leaq cond_nwaiters(%rdi), %rdi | ||
7019 | - movl $1, %edx | ||
7020 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
7021 | - movl $FUTEX_WAKE, %eax | ||
7022 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
7023 | - cmove %eax, %esi | ||
7024 | -#else | ||
7025 | - movl $0, %eax | ||
7026 | - movl %fs:PRIVATE_FUTEX, %esi | ||
7027 | - cmove %eax, %esi | ||
7028 | - orl $FUTEX_WAKE, %esi | ||
7029 | -#endif | ||
7030 | - movl $SYS_futex, %eax | ||
7031 | - syscall | ||
7032 | - subq $cond_nwaiters, %rdi | ||
7033 | - movl $1, %ecx | ||
7034 | - | ||
7035 | -4: LOCK | ||
7036 | -#if cond_lock == 0 | ||
7037 | - decl (%rdi) | ||
7038 | -#else | ||
7039 | - decl cond_lock(%rdi) | ||
7040 | -#endif | ||
7041 | - je 2f | ||
7042 | -#if cond_lock != 0 | ||
7043 | - addq $cond_lock, %rdi | ||
7044 | -#endif | ||
7045 | - LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi) | ||
7046 | - movl $LLL_PRIVATE, %eax | ||
7047 | - movl $LLL_SHARED, %esi | ||
7048 | - cmovne %eax, %esi | ||
7049 | - /* The call preserves %rcx. */ | ||
7050 | - callq __lll_unlock_wake | ||
7051 | - | ||
7052 | - /* Wake up all waiters to make sure no signal gets lost. */ | ||
7053 | -2: testl %ecx, %ecx | ||
7054 | - jnz 5f | ||
7055 | - addq $cond_futex, %rdi | ||
7056 | - LP_OP(cmp) $-1, dep_mutex-cond_futex(%rdi) | ||
7057 | - movl $0x7fffffff, %edx | ||
7058 | -#ifdef __ASSUME_PRIVATE_FUTEX | ||
7059 | - movl $FUTEX_WAKE, %eax | ||
7060 | - movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi | ||
7061 | - cmove %eax, %esi | ||
7062 | -#else | ||
7063 | - movl $0, %eax | ||
7064 | - movl %fs:PRIVATE_FUTEX, %esi | ||
7065 | - cmove %eax, %esi | ||
7066 | - orl $FUTEX_WAKE, %esi | ||
7067 | -#endif | ||
7068 | - movl $SYS_futex, %eax | ||
7069 | - syscall | ||
7070 | - | ||
7071 | - /* Lock the mutex only if we don't own it already. This only happens | ||
7072 | - in case of PI mutexes, if we got cancelled after a successful | ||
7073 | - return of the futex syscall and before disabling async | ||
7074 | - cancellation. */ | ||
7075 | -5: movq 16(%rsp), %rdi | ||
7076 | - movl MUTEX_KIND(%rdi), %eax | ||
7077 | - andl $(ROBUST_BIT|PI_BIT), %eax | ||
7078 | - cmpl $PI_BIT, %eax | ||
7079 | - jne 7f | ||
7080 | - | ||
7081 | - movl (%rdi), %eax | ||
7082 | - andl $TID_MASK, %eax | ||
7083 | - cmpl %eax, %fs:TID | ||
7084 | - jne 7f | ||
7085 | - /* We managed to get the lock. Fix it up before returning. */ | ||
7086 | - callq __pthread_mutex_cond_lock_adjust | ||
7087 | - jmp 8f | ||
7088 | - | ||
7089 | - | ||
7090 | -7: callq __pthread_mutex_cond_lock | ||
7091 | - | ||
7092 | -8: movq 24(%rsp), %rdi | ||
7093 | -.LcallUR: | ||
7094 | - call _Unwind_Resume | ||
7095 | - hlt | ||
7096 | -.LENDCODE: | ||
7097 | - cfi_endproc | ||
7098 | - .size __condvar_cleanup1, .-__condvar_cleanup1 | ||
7099 | - | ||
7100 | - | ||
7101 | - .section .gcc_except_table,"a",@progbits | ||
7102 | -.LexceptSTART: | ||
7103 | - .byte DW_EH_PE_omit # @LPStart format | ||
7104 | - .byte DW_EH_PE_omit # @TType format | ||
7105 | - .byte DW_EH_PE_uleb128 # call-site format | ||
7106 | - .uleb128 .Lcstend-.Lcstbegin | ||
7107 | -.Lcstbegin: | ||
7108 | - .uleb128 .LcleanupSTART-.LSTARTCODE | ||
7109 | - .uleb128 .LcleanupEND-.LcleanupSTART | ||
7110 | - .uleb128 __condvar_cleanup1-.LSTARTCODE | ||
7111 | - .uleb128 0 | ||
7112 | - .uleb128 .LcallUR-.LSTARTCODE | ||
7113 | - .uleb128 .LENDCODE-.LcallUR | ||
7114 | - .uleb128 0 | ||
7115 | - .uleb128 0 | ||
7116 | -.Lcstend: | ||
7117 | - | ||
7118 | - | ||
7119 | -#ifdef SHARED | ||
7120 | - .hidden DW.ref.__gcc_personality_v0 | ||
7121 | - .weak DW.ref.__gcc_personality_v0 | ||
7122 | - .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits | ||
7123 | - .align LP_SIZE | ||
7124 | - .type DW.ref.__gcc_personality_v0, @object | ||
7125 | - .size DW.ref.__gcc_personality_v0, LP_SIZE | ||
7126 | -DW.ref.__gcc_personality_v0: | ||
7127 | - ASM_ADDR __gcc_personality_v0 | ||
7128 | -#endif | ||
7129 | diff --git a/sysdeps/x86/bits/pthreadtypes.h b/sysdeps/x86/bits/pthreadtypes.h | ||
7130 | index 16b8f4f..a3a738f 100644 | ||
7131 | --- a/sysdeps/x86/bits/pthreadtypes.h | ||
7132 | +++ b/sysdeps/x86/bits/pthreadtypes.h | ||
7133 | @@ -140,14 +140,27 @@ typedef union | ||
7134 | { | ||
7135 | struct | ||
7136 | { | ||
7137 | - int __lock; | ||
7138 | - unsigned int __futex; | ||
7139 | - __extension__ unsigned long long int __total_seq; | ||
7140 | - __extension__ unsigned long long int __wakeup_seq; | ||
7141 | - __extension__ unsigned long long int __woken_seq; | ||
7142 | - void *__mutex; | ||
7143 | - unsigned int __nwaiters; | ||
7144 | - unsigned int __broadcast_seq; | ||
7145 | + __extension__ union | ||
7146 | + { | ||
7147 | + __extension__ unsigned long long int __wseq; | ||
7148 | + struct { | ||
7149 | + unsigned int __low; | ||
7150 | + unsigned int __high; | ||
7151 | + } __wseq32; | ||
7152 | + }; | ||
7153 | + __extension__ union | ||
7154 | + { | ||
7155 | + __extension__ unsigned long long int __g1_start; | ||
7156 | + struct { | ||
7157 | + unsigned int __low; | ||
7158 | + unsigned int __high; | ||
7159 | + } __g1_start32; | ||
7160 | + }; | ||
7161 | + unsigned int __g_refs[2]; | ||
7162 | + unsigned int __g_size[2]; | ||
7163 | + unsigned int __g1_orig_size; | ||
7164 | + unsigned int __wrefs; | ||
7165 | + unsigned int __g_signals[2]; | ||
7166 | } __data; | ||
7167 | char __size[__SIZEOF_PTHREAD_COND_T]; | ||
7168 | __extension__ long long int __align; | ||
7169 | -- | ||
7170 | 2.10.2 | ||
7171 | |||