summaryrefslogtreecommitdiffstats
path: root/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch
diff options
context:
space:
mode:
Diffstat (limited to 'meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch')
-rw-r--r--meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch7171
1 files changed, 7171 insertions, 0 deletions
diff --git a/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch b/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch
new file mode 100644
index 0000000000..3c7bfa160f
--- /dev/null
+++ b/meta/recipes-core/glibc/glibc/0004-New-condvar-implementation-that-provides-stronger-or.patch
@@ -0,0 +1,7171 @@
1From 27af8689a6ba8d182f3cbe6ba42cc654ceed0351 Mon Sep 17 00:00:00 2001
2From: Catalin Enache <catalin.enache@windriver.com>
3Date: Fri, 30 Jun 2017 11:56:41 +0300
4Subject: [PATCH 4/6] New condvar implementation that provides stronger
5 ordering guarantees.
6
7This is a new implementation for condition variables, required
8after http://austingroupbugs.net/view.php?id=609 to fix bug 13165. In
9essence, we need to be stricter in which waiters a signal or broadcast
10is required to wake up; this couldn't be solved using the old algorithm.
11ISO C++ made a similar clarification, so this also fixes a bug in
12current libstdc++, for example.
13
14We can't use the old algorithm anymore because futexes do not guarantee
15to wake in FIFO order. Thus, when we wake, we can't simply let any
16waiter grab a signal, but we need to ensure that one of the waiters
17happening before the signal is woken up. This is something the previous
18algorithm violated (see bug 13165).
19
20There's another issue specific to condvars: ABA issues on the underlying
21futexes. Unlike mutexes that have just three states, or semaphores that
22have no tokens or a limited number of them, the state of a condvar is
23the *order* of the waiters. A waiter on a semaphore can grab a token
24whenever one is available; a condvar waiter must only consume a signal
25if it is eligible to do so as determined by the relative order of the
26waiter and the signal.
27Therefore, this new algorithm maintains two groups of waiters: Those
28eligible to consume signals (G1), and those that have to wait until
29previous waiters have consumed signals (G2). Once G1 is empty, G2
30becomes the new G1. 64b counters are used to avoid ABA issues.
31
32This condvar doesn't yet use a requeue optimization (ie, on a broadcast,
33waking just one thread and requeueing all others on the futex of the
34mutex supplied by the program). I don't think doing the requeue is
35necessarily the right approach (but I haven't done real measurements
36yet):
37* If a program expects to wake many threads at the same time and make
38that scalable, a condvar isn't great anyway because of how it requires
39waiters to operate mutually exclusive (due to the mutex usage). Thus, a
40thundering herd problem is a scalability problem with or without the
41optimization. Using something like a semaphore might be more
42appropriate in such a case.
43* The scalability problem is actually at the mutex side; the condvar
44could help (and it tries to with the requeue optimization), but it
45should be the mutex who decides how that is done, and whether it is done
46at all.
47* Forcing all but one waiter into the kernel-side wait queue of the
48mutex prevents/avoids the use of lock elision on the mutex. Thus, it
49prevents the only cure against the underlying scalability problem
50inherent to condvars.
51* If condvars use short critical sections (ie, hold the mutex just to
52check a binary flag or such), which they should do ideally, then forcing
53all those waiter to proceed serially with kernel-based hand-off (ie,
54futex ops in the mutex' contended state, via the futex wait queues) will
55be less efficient than just letting a scalable mutex implementation take
56care of it. Our current mutex impl doesn't employ spinning at all, but
57if critical sections are short, spinning can be much better.
58* Doing the requeue stuff requires all waiters to always drive the mutex
59into the contended state. This leads to each waiter having to call
60futex_wake after lock release, even if this wouldn't be necessary.
61
62 [BZ #13165]
63 * nptl/pthread_cond_broadcast.c (__pthread_cond_broadcast): Rewrite to
64 use new algorithm.
65 * nptl/pthread_cond_destroy.c (__pthread_cond_destroy): Likewise.
66 * nptl/pthread_cond_init.c (__pthread_cond_init): Likewise.
67 * nptl/pthread_cond_signal.c (__pthread_cond_signal): Likewise.
68 * nptl/pthread_cond_wait.c (__pthread_cond_wait): Likewise.
69 (__pthread_cond_timedwait): Move here from pthread_cond_timedwait.c.
70 (__condvar_confirm_wakeup, __condvar_cancel_waiting,
71 __condvar_cleanup_waiting, __condvar_dec_grefs,
72 __pthread_cond_wait_common): New.
73 (__condvar_cleanup): Remove.
74 * npt/pthread_condattr_getclock.c (pthread_condattr_getclock): Adapt.
75 * npt/pthread_condattr_setclock.c (pthread_condattr_setclock):
76 Likewise.
77 * npt/pthread_condattr_getpshared.c (pthread_condattr_getpshared):
78 Likewise.
79 * npt/pthread_condattr_init.c (pthread_condattr_init): Likewise.
80 * nptl/tst-cond1.c: Add comment.
81 * nptl/tst-cond20.c (do_test): Adapt.
82 * nptl/tst-cond22.c (do_test): Likewise.
83 * sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_cond_t): Adapt
84 structure.
85 * sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
86 * sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
87 * sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
88 * sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_cond_t):
89 Likewise.
90 * sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
91 * sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
92 * sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
93 * sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
94 * sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
95 * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_cond_t):
96 Likewise.
97 * sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h (pthread_cond_t):
98 Likewise.
99 * sysdeps/x86/bits/pthreadtypes.h (pthread_cond_t): Likewise.
100 * sysdeps/nptl/internaltypes.h (COND_NWAITERS_SHIFT): Remove.
101 (COND_CLOCK_BITS): Adapt.
102 * sysdeps/nptl/pthread.h (PTHREAD_COND_INITIALIZER): Adapt.
103 * nptl/pthreadP.h (__PTHREAD_COND_CLOCK_MONOTONIC_MASK,
104 __PTHREAD_COND_SHARED_MASK): New.
105 * nptl/nptl-printers.py (CLOCK_IDS): Remove.
106 (ConditionVariablePrinter, ConditionVariableAttributesPrinter): Adapt.
107 * nptl/nptl_lock_constants.pysym: Adapt.
108 * nptl/test-cond-printers.py: Adapt.
109 * sysdeps/unix/sysv/linux/hppa/internaltypes.h (cond_compat_clear,
110 cond_compat_check_and_clear): Adapt.
111 * sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c: Remove file ...
112 * sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c
113 (__pthread_cond_timedwait): ... and move here.
114 * nptl/DESIGN-condvar.txt: Remove file.
115 * nptl/lowlevelcond.sym: Likewise.
116 * nptl/pthread_cond_timedwait.c: Likewise.
117 * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise.
118 * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
119 * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
120 * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
121 * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_broadcast.S: Likewise.
122 * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_signal.S: Likewise.
123 * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_timedwait.S: Likewise.
124 * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_wait.S: Likewise.
125 * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_broadcast.S: Likewise.
126 * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_signal.S: Likewise.
127 * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S: Likewise.
128 * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_wait.S: Likewise.
129 * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise.
130 * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
131 * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
132 * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
133
134Upstream-Status: Backport
135
136Author: Torvald Riegel <triegel@redhat.com>
137Signed-off-by: Catalin Enache <catalin.enache@windriver.com>
138---
139 ChangeLog | 74 ++
140 nptl/DESIGN-condvar.txt | 134 ---
141 nptl/Makefile | 6 +-
142 nptl/lowlevelcond.sym | 16 -
143 nptl/nptl-printers.py | 70 +-
144 nptl/nptl_lock_constants.pysym | 27 +-
145 nptl/pthreadP.h | 7 +
146 nptl/pthread_cond_broadcast.c | 99 ++-
147 nptl/pthread_cond_common.c | 466 ++++++++++
148 nptl/pthread_cond_destroy.c | 82 +-
149 nptl/pthread_cond_init.c | 28 +-
150 nptl/pthread_cond_signal.c | 99 ++-
151 nptl/pthread_cond_timedwait.c | 268 ------
152 nptl/pthread_cond_wait.c | 754 ++++++++++++----
153 nptl/pthread_condattr_getclock.c | 2 +-
154 nptl/pthread_condattr_getpshared.c | 3 +-
155 nptl/pthread_condattr_init.c | 4 +-
156 nptl/pthread_condattr_setclock.c | 11 +-
157 nptl/test-cond-printers.py | 2 +-
158 nptl/tst-cond1.c | 3 +
159 nptl/tst-cond20.c | 5 +-
160 nptl/tst-cond22.c | 18 +-
161 sysdeps/aarch64/nptl/bits/pthreadtypes.h | 31 +-
162 sysdeps/arm/nptl/bits/pthreadtypes.h | 29 +-
163 sysdeps/ia64/nptl/bits/pthreadtypes.h | 31 +-
164 sysdeps/m68k/nptl/bits/pthreadtypes.h | 32 +-
165 sysdeps/microblaze/nptl/bits/pthreadtypes.h | 29 +-
166 sysdeps/mips/nptl/bits/pthreadtypes.h | 31 +-
167 sysdeps/nios2/nptl/bits/pthreadtypes.h | 31 +-
168 sysdeps/nptl/internaltypes.h | 17 +-
169 sysdeps/nptl/pthread.h | 2 +-
170 sysdeps/s390/nptl/bits/pthreadtypes.h | 29 +-
171 sysdeps/sh/nptl/bits/pthreadtypes.h | 29 +-
172 sysdeps/tile/nptl/bits/pthreadtypes.h | 29 +-
173 sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h | 31 +-
174 sysdeps/unix/sysv/linux/hppa/internaltypes.h | 40 +-
175 .../unix/sysv/linux/hppa/pthread_cond_timedwait.c | 41 -
176 sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c | 13 +
177 .../sysv/linux/i386/i686/pthread_cond_timedwait.S | 20 -
178 .../unix/sysv/linux/i386/pthread_cond_broadcast.S | 241 -----
179 sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S | 216 -----
180 .../unix/sysv/linux/i386/pthread_cond_timedwait.S | 974 ---------------------
181 sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S | 642 --------------
182 .../unix/sysv/linux/powerpc/bits/pthreadtypes.h | 31 +-
183 .../sysv/linux/x86_64/pthread_cond_broadcast.S | 177 ----
184 .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 161 ----
185 .../sysv/linux/x86_64/pthread_cond_timedwait.S | 623 -------------
186 sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S | 555 ------------
187 sysdeps/x86/bits/pthreadtypes.h | 29 +-
188 49 files changed, 1671 insertions(+), 4621 deletions(-)
189 delete mode 100644 nptl/DESIGN-condvar.txt
190 delete mode 100644 nptl/lowlevelcond.sym
191 create mode 100644 nptl/pthread_cond_common.c
192 delete mode 100644 nptl/pthread_cond_timedwait.c
193 delete mode 100644 sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c
194 delete mode 100644 sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S
195 delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S
196 delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S
197 delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S
198 delete mode 100644 sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S
199 delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
200 delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
201 delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
202 delete mode 100644 sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
203
204diff --git a/ChangeLog b/ChangeLog
205index 8036c1e..c94db7b 100644
206--- a/ChangeLog
207+++ b/ChangeLog
208@@ -1,3 +1,77 @@
209+2016-12-31 Torvald Riegel <triegel@redhat.com>
210+
211+ [BZ #13165]
212+ * nptl/pthread_cond_broadcast.c (__pthread_cond_broadcast): Rewrite to
213+ use new algorithm.
214+ * nptl/pthread_cond_destroy.c (__pthread_cond_destroy): Likewise.
215+ * nptl/pthread_cond_init.c (__pthread_cond_init): Likewise.
216+ * nptl/pthread_cond_signal.c (__pthread_cond_signal): Likewise.
217+ * nptl/pthread_cond_wait.c (__pthread_cond_wait): Likewise.
218+ (__pthread_cond_timedwait): Move here from pthread_cond_timedwait.c.
219+ (__condvar_confirm_wakeup, __condvar_cancel_waiting,
220+ __condvar_cleanup_waiting, __condvar_dec_grefs,
221+ __pthread_cond_wait_common): New.
222+ (__condvar_cleanup): Remove.
223+ * npt/pthread_condattr_getclock.c (pthread_condattr_getclock): Adapt.
224+ * npt/pthread_condattr_setclock.c (pthread_condattr_setclock):
225+ Likewise.
226+ * npt/pthread_condattr_getpshared.c (pthread_condattr_getpshared):
227+ Likewise.
228+ * npt/pthread_condattr_init.c (pthread_condattr_init): Likewise.
229+ * nptl/tst-cond1.c: Add comment.
230+ * nptl/tst-cond20.c (do_test): Adapt.
231+ * nptl/tst-cond22.c (do_test): Likewise.
232+ * sysdeps/aarch64/nptl/bits/pthreadtypes.h (pthread_cond_t): Adapt
233+ structure.
234+ * sysdeps/arm/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
235+ * sysdeps/ia64/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
236+ * sysdeps/m68k/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
237+ * sysdeps/microblaze/nptl/bits/pthreadtypes.h (pthread_cond_t):
238+ Likewise.
239+ * sysdeps/mips/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
240+ * sysdeps/nios2/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
241+ * sysdeps/s390/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
242+ * sysdeps/sh/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
243+ * sysdeps/tile/nptl/bits/pthreadtypes.h (pthread_cond_t): Likewise.
244+ * sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h (pthread_cond_t):
245+ Likewise.
246+ * sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h (pthread_cond_t):
247+ Likewise.
248+ * sysdeps/x86/bits/pthreadtypes.h (pthread_cond_t): Likewise.
249+ * sysdeps/nptl/internaltypes.h (COND_NWAITERS_SHIFT): Remove.
250+ (COND_CLOCK_BITS): Adapt.
251+ * sysdeps/nptl/pthread.h (PTHREAD_COND_INITIALIZER): Adapt.
252+ * nptl/pthreadP.h (__PTHREAD_COND_CLOCK_MONOTONIC_MASK,
253+ __PTHREAD_COND_SHARED_MASK): New.
254+ * nptl/nptl-printers.py (CLOCK_IDS): Remove.
255+ (ConditionVariablePrinter, ConditionVariableAttributesPrinter): Adapt.
256+ * nptl/nptl_lock_constants.pysym: Adapt.
257+ * nptl/test-cond-printers.py: Adapt.
258+ * sysdeps/unix/sysv/linux/hppa/internaltypes.h (cond_compat_clear,
259+ cond_compat_check_and_clear): Adapt.
260+ * sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c: Remove file ...
261+ * sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c
262+ (__pthread_cond_timedwait): ... and move here.
263+ * nptl/DESIGN-condvar.txt: Remove file.
264+ * nptl/lowlevelcond.sym: Likewise.
265+ * nptl/pthread_cond_timedwait.c: Likewise.
266+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: Likewise.
267+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
268+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: Likewise.
269+ * sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
270+ * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_broadcast.S: Likewise.
271+ * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_signal.S: Likewise.
272+ * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_timedwait.S: Likewise.
273+ * sysdeps/unix/sysv/linux/i386/i586/pthread_cond_wait.S: Likewise.
274+ * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_broadcast.S: Likewise.
275+ * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_signal.S: Likewise.
276+ * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S: Likewise.
277+ * sysdeps/unix/sysv/linux/i386/i686/pthread_cond_wait.S: Likewise.
278+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S: Likewise.
279+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S: Likewise.
280+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S: Likewise.
281+ * sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S: Likewise.
282+
283 2016-12-08 Martin Galvan <martin.galvan@tallertechnologies.com>
284
285 * INSTALL: Regenerated.
286diff --git a/nptl/DESIGN-condvar.txt b/nptl/DESIGN-condvar.txt
287deleted file mode 100644
288index 4845251..0000000
289--- a/nptl/DESIGN-condvar.txt
290+++ /dev/null
291@@ -1,134 +0,0 @@
292-Conditional Variable pseudocode.
293-================================
294-
295- int pthread_cond_timedwait (pthread_cond_t *cv, pthread_mutex_t *mutex);
296- int pthread_cond_signal (pthread_cond_t *cv);
297- int pthread_cond_broadcast (pthread_cond_t *cv);
298-
299-struct pthread_cond_t {
300-
301- unsigned int cond_lock;
302-
303- internal mutex
304-
305- uint64_t total_seq;
306-
307- Total number of threads using the conditional variable.
308-
309- uint64_t wakeup_seq;
310-
311- sequence number for next wakeup.
312-
313- uint64_t woken_seq;
314-
315- sequence number of last woken thread.
316-
317- uint32_t broadcast_seq;
318-
319-}
320-
321-
322-struct cv_data {
323-
324- pthread_cond_t *cv;
325-
326- uint32_t bc_seq
327-
328-}
329-
330-
331-
332-cleanup_handler(cv_data)
333-{
334- cv = cv_data->cv;
335- lll_lock(cv->lock);
336-
337- if (cv_data->bc_seq == cv->broadcast_seq) {
338- ++cv->wakeup_seq;
339- ++cv->woken_seq;
340- }
341-
342- /* make sure no signal gets lost. */
343- FUTEX_WAKE(cv->wakeup_seq, ALL);
344-
345- lll_unlock(cv->lock);
346-}
347-
348-
349-cond_timedwait(cv, mutex, timeout):
350-{
351- lll_lock(cv->lock);
352- mutex_unlock(mutex);
353-
354- cleanup_push
355-
356- ++cv->total_seq;
357- val = seq = cv->wakeup_seq;
358- cv_data.bc = cv->broadcast_seq;
359- cv_data.cv = cv;
360-
361- while (1) {
362-
363- lll_unlock(cv->lock);
364-
365- enable_async(&cv_data);
366-
367- ret = FUTEX_WAIT(cv->wakeup_seq, val, timeout);
368-
369- restore_async
370-
371- lll_lock(cv->lock);
372-
373- if (bc != cv->broadcast_seq)
374- goto bc_out;
375-
376- val = cv->wakeup_seq;
377-
378- if (val != seq && cv->woken_seq != val) {
379- ret = 0;
380- break;
381- }
382-
383- if (ret == TIMEDOUT) {
384- ++cv->wakeup_seq;
385- break;
386- }
387- }
388-
389- ++cv->woken_seq;
390-
391- bc_out:
392- lll_unlock(cv->lock);
393-
394- cleanup_pop
395-
396- mutex_lock(mutex);
397-
398- return ret;
399-}
400-
401-cond_signal(cv)
402-{
403- lll_lock(cv->lock);
404-
405- if (cv->total_seq > cv->wakeup_seq) {
406- ++cv->wakeup_seq;
407- FUTEX_WAKE(cv->wakeup_seq, 1);
408- }
409-
410- lll_unlock(cv->lock);
411-}
412-
413-cond_broadcast(cv)
414-{
415- lll_lock(cv->lock);
416-
417- if (cv->total_seq > cv->wakeup_seq) {
418- cv->wakeup_seq = cv->total_seq;
419- cv->woken_seq = cv->total_seq;
420- ++cv->broadcast_seq;
421- FUTEX_WAKE(cv->wakeup_seq, ALL);
422- }
423-
424- lll_unlock(cv->lock);
425-}
426diff --git a/nptl/Makefile b/nptl/Makefile
427index 49f6ba6..1f0674c 100644
428--- a/nptl/Makefile
429+++ b/nptl/Makefile
430@@ -71,7 +71,7 @@ libpthread-routines = nptl-init vars events version pt-interp \
431 pthread_rwlockattr_getkind_np \
432 pthread_rwlockattr_setkind_np \
433 pthread_cond_init pthread_cond_destroy \
434- pthread_cond_wait pthread_cond_timedwait \
435+ pthread_cond_wait \
436 pthread_cond_signal pthread_cond_broadcast \
437 old_pthread_cond_init old_pthread_cond_destroy \
438 old_pthread_cond_wait old_pthread_cond_timedwait \
439@@ -181,7 +181,6 @@ CFLAGS-pthread_timedjoin.c = -fexceptions -fasynchronous-unwind-tables
440 CFLAGS-pthread_once.c = $(uses-callbacks) -fexceptions \
441 -fasynchronous-unwind-tables
442 CFLAGS-pthread_cond_wait.c = -fexceptions -fasynchronous-unwind-tables
443-CFLAGS-pthread_cond_timedwait.c = -fexceptions -fasynchronous-unwind-tables
444 CFLAGS-sem_wait.c = -fexceptions -fasynchronous-unwind-tables
445 CFLAGS-sem_timedwait.c = -fexceptions -fasynchronous-unwind-tables
446
447@@ -303,8 +302,7 @@ test-xfail-tst-once5 = yes
448 # Files which must not be linked with libpthread.
449 tests-nolibpthread = tst-unload
450
451-gen-as-const-headers = pthread-errnos.sym \
452- lowlevelcond.sym lowlevelrwlock.sym \
453+gen-as-const-headers = pthread-errnos.sym lowlevelrwlock.sym \
454 unwindbuf.sym \
455 lowlevelrobustlock.sym pthread-pi-defines.sym
456
457diff --git a/nptl/lowlevelcond.sym b/nptl/lowlevelcond.sym
458deleted file mode 100644
459index 18e1ada..0000000
460--- a/nptl/lowlevelcond.sym
461+++ /dev/null
462@@ -1,16 +0,0 @@
463-#include <stddef.h>
464-#include <sched.h>
465-#include <bits/pthreadtypes.h>
466-#include <internaltypes.h>
467-
468---
469-
470-cond_lock offsetof (pthread_cond_t, __data.__lock)
471-cond_futex offsetof (pthread_cond_t, __data.__futex)
472-cond_nwaiters offsetof (pthread_cond_t, __data.__nwaiters)
473-total_seq offsetof (pthread_cond_t, __data.__total_seq)
474-wakeup_seq offsetof (pthread_cond_t, __data.__wakeup_seq)
475-woken_seq offsetof (pthread_cond_t, __data.__woken_seq)
476-dep_mutex offsetof (pthread_cond_t, __data.__mutex)
477-broadcast_seq offsetof (pthread_cond_t, __data.__broadcast_seq)
478-nwaiters_shift COND_NWAITERS_SHIFT
479diff --git a/nptl/nptl-printers.py b/nptl/nptl-printers.py
480index e402f23..76adadd 100644
481--- a/nptl/nptl-printers.py
482+++ b/nptl/nptl-printers.py
483@@ -293,16 +293,6 @@ class MutexAttributesPrinter(object):
484 elif protocol == PTHREAD_PRIO_PROTECT:
485 self.values.append(('Protocol', 'Priority protect'))
486
487-CLOCK_IDS = {
488- CLOCK_REALTIME: 'CLOCK_REALTIME',
489- CLOCK_MONOTONIC: 'CLOCK_MONOTONIC',
490- CLOCK_PROCESS_CPUTIME_ID: 'CLOCK_PROCESS_CPUTIME_ID',
491- CLOCK_THREAD_CPUTIME_ID: 'CLOCK_THREAD_CPUTIME_ID',
492- CLOCK_MONOTONIC_RAW: 'CLOCK_MONOTONIC_RAW',
493- CLOCK_REALTIME_COARSE: 'CLOCK_REALTIME_COARSE',
494- CLOCK_MONOTONIC_COARSE: 'CLOCK_MONOTONIC_COARSE'
495-}
496-
497 class ConditionVariablePrinter(object):
498 """Pretty printer for pthread_cond_t."""
499
500@@ -313,24 +303,8 @@ class ConditionVariablePrinter(object):
501 cond: A gdb.value representing a pthread_cond_t.
502 """
503
504- # Since PTHREAD_COND_SHARED is an integer, we need to cast it to void *
505- # to be able to compare it to the condvar's __data.__mutex member.
506- #
507- # While it looks like self.shared_value should be a class variable,
508- # that would result in it having an incorrect size if we're loading
509- # these printers through .gdbinit for a 64-bit objfile in AMD64.
510- # This is because gdb initially assumes the pointer size to be 4 bytes,
511- # and only sets it to 8 after loading the 64-bit objfiles. Since
512- # .gdbinit runs before any objfiles are loaded, this would effectively
513- # make self.shared_value have a size of 4, thus breaking later
514- # comparisons with pointers whose types are looked up at runtime.
515- void_ptr_type = gdb.lookup_type('void').pointer()
516- self.shared_value = gdb.Value(PTHREAD_COND_SHARED).cast(void_ptr_type)
517-
518 data = cond['__data']
519- self.total_seq = data['__total_seq']
520- self.mutex = data['__mutex']
521- self.nwaiters = data['__nwaiters']
522+ self.wrefs = data['__wrefs']
523 self.values = []
524
525 self.read_values()
526@@ -360,7 +334,6 @@ class ConditionVariablePrinter(object):
527
528 self.read_status()
529 self.read_attributes()
530- self.read_mutex_info()
531
532 def read_status(self):
533 """Read the status of the condvar.
534@@ -369,41 +342,22 @@ class ConditionVariablePrinter(object):
535 are waiting for it.
536 """
537
538- if self.total_seq == PTHREAD_COND_DESTROYED:
539- self.values.append(('Status', 'Destroyed'))
540-
541- self.values.append(('Threads waiting for this condvar',
542- self.nwaiters >> COND_NWAITERS_SHIFT))
543+ self.values.append(('Threads known to still execute a wait function',
544+ self.wrefs >> PTHREAD_COND_WREFS_SHIFT))
545
546 def read_attributes(self):
547 """Read the condvar's attributes."""
548
549- clock_id = self.nwaiters & ((1 << COND_NWAITERS_SHIFT) - 1)
550-
551- # clock_id must be casted to int because it's a gdb.Value
552- self.values.append(('Clock ID', CLOCK_IDS[int(clock_id)]))
553+ if (self.wrefs & PTHREAD_COND_CLOCK_MONOTONIC_MASK) != 0:
554+ self.values.append(('Clock ID', 'CLOCK_MONOTONIC'))
555+ else:
556+ self.values.append(('Clock ID', 'CLOCK_REALTIME'))
557
558- shared = (self.mutex == self.shared_value)
559-
560- if shared:
561+ if (self.wrefs & PTHREAD_COND_SHARED_MASK) != 0:
562 self.values.append(('Shared', 'Yes'))
563 else:
564 self.values.append(('Shared', 'No'))
565
566- def read_mutex_info(self):
567- """Read the data of the mutex this condvar is bound to.
568-
569- A pthread_cond_t's __data.__mutex member is a void * which
570- must be casted to pthread_mutex_t *. For shared condvars, this
571- member isn't recorded and has a special value instead.
572- """
573-
574- if self.mutex and self.mutex != self.shared_value:
575- mutex_type = gdb.lookup_type('pthread_mutex_t')
576- mutex = self.mutex.cast(mutex_type.pointer()).dereference()
577-
578- self.values.append(('Mutex', mutex))
579-
580 class ConditionVariableAttributesPrinter(object):
581 """Pretty printer for pthread_condattr_t.
582
583@@ -453,10 +407,12 @@ class ConditionVariableAttributesPrinter(object):
584 created in self.children.
585 """
586
587- clock_id = self.condattr & ((1 << COND_NWAITERS_SHIFT) - 1)
588+ clock_id = (self.condattr >> 1) & ((1 << COND_CLOCK_BITS) - 1)
589
590- # clock_id must be casted to int because it's a gdb.Value
591- self.values.append(('Clock ID', CLOCK_IDS[int(clock_id)]))
592+ if clock_id != 0:
593+ self.values.append(('Clock ID', 'CLOCK_MONOTONIC'))
594+ else:
595+ self.values.append(('Clock ID', 'CLOCK_REALTIME'))
596
597 if self.condattr & 1:
598 self.values.append(('Shared', 'Yes'))
599diff --git a/nptl/nptl_lock_constants.pysym b/nptl/nptl_lock_constants.pysym
600index 303ec61..2ab3179 100644
601--- a/nptl/nptl_lock_constants.pysym
602+++ b/nptl/nptl_lock_constants.pysym
603@@ -44,26 +44,13 @@ PTHREAD_PRIO_NONE
604 PTHREAD_PRIO_INHERIT
605 PTHREAD_PRIO_PROTECT
606
607--- These values are hardcoded as well:
608--- Value of __mutex for shared condvars.
609-PTHREAD_COND_SHARED (void *)~0l
610-
611--- Value of __total_seq for destroyed condvars.
612-PTHREAD_COND_DESTROYED -1ull
613-
614--- __nwaiters encodes the number of threads waiting on a condvar
615--- and the clock ID.
616--- __nwaiters >> COND_NWAITERS_SHIFT gives us the number of waiters.
617-COND_NWAITERS_SHIFT
618-
619--- Condvar clock IDs
620-CLOCK_REALTIME
621-CLOCK_MONOTONIC
622-CLOCK_PROCESS_CPUTIME_ID
623-CLOCK_THREAD_CPUTIME_ID
624-CLOCK_MONOTONIC_RAW
625-CLOCK_REALTIME_COARSE
626-CLOCK_MONOTONIC_COARSE
627+-- Condition variable
628+-- FIXME Why do macros prefixed with __ cannot be used directly?
629+PTHREAD_COND_SHARED_MASK __PTHREAD_COND_SHARED_MASK
630+PTHREAD_COND_CLOCK_MONOTONIC_MASK __PTHREAD_COND_CLOCK_MONOTONIC_MASK
631+COND_CLOCK_BITS
632+-- These values are hardcoded:
633+PTHREAD_COND_WREFS_SHIFT 3
634
635 -- Rwlock attributes
636 PTHREAD_RWLOCK_PREFER_READER_NP
637diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
638index 4edc74b..e9992bc 100644
639--- a/nptl/pthreadP.h
640+++ b/nptl/pthreadP.h
641@@ -167,6 +167,13 @@ enum
642 #define __PTHREAD_ONCE_FORK_GEN_INCR 4
643
644
645+/* Condition variable definitions. See __pthread_cond_wait_common.
646+ Need to be defined here so there is one place from which
647+ nptl_lock_constants can grab them. */
648+#define __PTHREAD_COND_CLOCK_MONOTONIC_MASK 2
649+#define __PTHREAD_COND_SHARED_MASK 1
650+
651+
652 /* Internal variables. */
653
654
655diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c
656index 552fd42..87c0755 100644
657--- a/nptl/pthread_cond_broadcast.c
658+++ b/nptl/pthread_cond_broadcast.c
659@@ -19,72 +19,71 @@
660 #include <endian.h>
661 #include <errno.h>
662 #include <sysdep.h>
663-#include <lowlevellock.h>
664+#include <futex-internal.h>
665 #include <pthread.h>
666 #include <pthreadP.h>
667 #include <stap-probe.h>
668+#include <atomic.h>
669
670 #include <shlib-compat.h>
671-#include <kernel-features.h>
672
673+#include "pthread_cond_common.c"
674
675+
676+/* We do the following steps from __pthread_cond_signal in one critical
677+ section: (1) signal all waiters in G1, (2) close G1 so that it can become
678+ the new G2 and make G2 the new G1, and (3) signal all waiters in the new
679+ G1. We don't need to do all these steps if there are no waiters in G1
680+ and/or G2. See __pthread_cond_signal for further details. */
681 int
682 __pthread_cond_broadcast (pthread_cond_t *cond)
683 {
684 LIBC_PROBE (cond_broadcast, 1, cond);
685
686- int pshared = (cond->__data.__mutex == (void *) ~0l)
687- ? LLL_SHARED : LLL_PRIVATE;
688- /* Make sure we are alone. */
689- lll_lock (cond->__data.__lock, pshared);
690+ unsigned int wrefs = atomic_load_relaxed (&cond->__data.__wrefs);
691+ if (wrefs >> 3 == 0)
692+ return 0;
693+ int private = __condvar_get_private (wrefs);
694+
695+ __condvar_acquire_lock (cond, private);
696
697- /* Are there any waiters to be woken? */
698- if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
699+ unsigned long long int wseq = __condvar_load_wseq_relaxed (cond);
700+ unsigned int g2 = wseq & 1;
701+ unsigned int g1 = g2 ^ 1;
702+ wseq >>= 1;
703+ bool do_futex_wake = false;
704+
705+ /* Step (1): signal all waiters remaining in G1. */
706+ if (cond->__data.__g_size[g1] != 0)
707 {
708- /* Yes. Mark them all as woken. */
709- cond->__data.__wakeup_seq = cond->__data.__total_seq;
710- cond->__data.__woken_seq = cond->__data.__total_seq;
711- cond->__data.__futex = (unsigned int) cond->__data.__total_seq * 2;
712- int futex_val = cond->__data.__futex;
713- /* Signal that a broadcast happened. */
714- ++cond->__data.__broadcast_seq;
715-
716- /* We are done. */
717- lll_unlock (cond->__data.__lock, pshared);
718-
719- /* Wake everybody. */
720- pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
721-
722- /* Do not use requeue for pshared condvars. */
723- if (mut == (void *) ~0l
724- || PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT)
725- goto wake_all;
726-
727-#if (defined lll_futex_cmp_requeue_pi \
728- && defined __ASSUME_REQUEUE_PI)
729- if (USE_REQUEUE_PI (mut))
730- {
731- if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX,
732- &mut->__data.__lock, futex_val,
733- LLL_PRIVATE) == 0)
734- return 0;
735- }
736- else
737-#endif
738- /* lll_futex_requeue returns 0 for success and non-zero
739- for errors. */
740- if (!__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
741- INT_MAX, &mut->__data.__lock,
742- futex_val, LLL_PRIVATE), 0))
743- return 0;
744-
745-wake_all:
746- lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
747- return 0;
748+ /* Add as many signals as the remaining size of the group. */
749+ atomic_fetch_add_relaxed (cond->__data.__g_signals + g1,
750+ cond->__data.__g_size[g1] << 1);
751+ cond->__data.__g_size[g1] = 0;
752+
753+ /* We need to wake G1 waiters before we quiesce G1 below. */
754+ /* TODO Only set it if there are indeed futex waiters. We could
755+ also try to move this out of the critical section in cases when
756+ G2 is empty (and we don't need to quiesce). */
757+ futex_wake (cond->__data.__g_signals + g1, INT_MAX, private);
758 }
759
760- /* We are done. */
761- lll_unlock (cond->__data.__lock, pshared);
762+ /* G1 is complete. Step (2) is next unless there are no waiters in G2, in
763+ which case we can stop. */
764+ if (__condvar_quiesce_and_switch_g1 (cond, wseq, &g1, private))
765+ {
766+ /* Step (3): Send signals to all waiters in the old G2 / new G1. */
767+ atomic_fetch_add_relaxed (cond->__data.__g_signals + g1,
768+ cond->__data.__g_size[g1] << 1);
769+ cond->__data.__g_size[g1] = 0;
770+ /* TODO Only set it if there are indeed futex waiters. */
771+ do_futex_wake = true;
772+ }
773+
774+ __condvar_release_lock (cond, private);
775+
776+ if (do_futex_wake)
777+ futex_wake (cond->__data.__g_signals + g1, INT_MAX, private);
778
779 return 0;
780 }
781diff --git a/nptl/pthread_cond_common.c b/nptl/pthread_cond_common.c
782new file mode 100644
783index 0000000..b374396
784--- /dev/null
785+++ b/nptl/pthread_cond_common.c
786@@ -0,0 +1,466 @@
787+/* pthread_cond_common -- shared code for condition variable.
788+ Copyright (C) 2016 Free Software Foundation, Inc.
789+ This file is part of the GNU C Library.
790+
791+ The GNU C Library is free software; you can redistribute it and/or
792+ modify it under the terms of the GNU Lesser General Public
793+ License as published by the Free Software Foundation; either
794+ version 2.1 of the License, or (at your option) any later version.
795+
796+ The GNU C Library is distributed in the hope that it will be useful,
797+ but WITHOUT ANY WARRANTY; without even the implied warranty of
798+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
799+ Lesser General Public License for more details.
800+
801+ You should have received a copy of the GNU Lesser General Public
802+ License along with the GNU C Library; if not, see
803+ <http://www.gnu.org/licenses/>. */
804+
805+#include <atomic.h>
806+#include <stdint.h>
807+#include <pthread.h>
808+#include <libc-internal.h>
809+
810+/* We need 3 least-significant bits on __wrefs for something else. */
811+#define __PTHREAD_COND_MAX_GROUP_SIZE ((unsigned) 1 << 29)
812+
813+#if __HAVE_64B_ATOMICS == 1
814+
815+static uint64_t __attribute__ ((unused))
816+__condvar_load_wseq_relaxed (pthread_cond_t *cond)
817+{
818+ return atomic_load_relaxed (&cond->__data.__wseq);
819+}
820+
821+static uint64_t __attribute__ ((unused))
822+__condvar_fetch_add_wseq_acquire (pthread_cond_t *cond, unsigned int val)
823+{
824+ return atomic_fetch_add_acquire (&cond->__data.__wseq, val);
825+}
826+
827+static uint64_t __attribute__ ((unused))
828+__condvar_fetch_xor_wseq_release (pthread_cond_t *cond, unsigned int val)
829+{
830+ return atomic_fetch_xor_release (&cond->__data.__wseq, val);
831+}
832+
833+static uint64_t __attribute__ ((unused))
834+__condvar_load_g1_start_relaxed (pthread_cond_t *cond)
835+{
836+ return atomic_load_relaxed (&cond->__data.__g1_start);
837+}
838+
839+static void __attribute__ ((unused))
840+__condvar_add_g1_start_relaxed (pthread_cond_t *cond, unsigned int val)
841+{
842+ atomic_store_relaxed (&cond->__data.__g1_start,
843+ atomic_load_relaxed (&cond->__data.__g1_start) + val);
844+}
845+
846+#else
847+
848+/* We use two 64b counters: __wseq and __g1_start. They are monotonically
849+ increasing and single-writer-multiple-readers counters, so we can implement
850+ load, fetch-and-add, and fetch-and-xor operations even when we just have
851+ 32b atomics. Values we add or xor are less than or equal to 1<<31 (*),
852+ so we only have to make overflow-and-addition atomic wrt. to concurrent
853+ load operations and xor operations. To do that, we split each counter into
854+ two 32b values of which we reserve the MSB of each to represent an
855+ overflow from the lower-order half to the higher-order half.
856+
857+ In the common case, the state is (higher-order / lower-order half, and . is
858+ basically concatenation of the bits):
859+ 0.h / 0.l = h.l
860+
861+ When we add a value of x that overflows (i.e., 0.l + x == 1.L), we run the
862+ following steps S1-S4 (the values these represent are on the right-hand
863+ side):
864+ S1: 0.h / 1.L == (h+1).L
865+ S2: 1.(h+1) / 1.L == (h+1).L
866+ S3: 1.(h+1) / 0.L == (h+1).L
867+ S4: 0.(h+1) / 0.L == (h+1).L
868+ If the LSB of the higher-order half is set, readers will ignore the
869+ overflow bit in the lower-order half.
870+
871+ To get an atomic snapshot in load operations, we exploit that the
872+ higher-order half is monotonically increasing; if we load a value V from
873+ it, then read the lower-order half, and then read the higher-order half
874+ again and see the same value V, we know that both halves have existed in
875+ the sequence of values the full counter had. This is similar to the
876+ validated reads in the time-based STMs in GCC's libitm (e.g.,
877+ method_ml_wt).
878+
879+ The xor operation needs to be an atomic read-modify-write. The write
880+ itself is not an issue as it affects just the lower-order half but not bits
881+ used in the add operation. To make the full fetch-and-xor atomic, we
882+ exploit that concurrently, the value can increase by at most 1<<31 (*): The
883+ xor operation is only called while having acquired the lock, so not more
884+ than __PTHREAD_COND_MAX_GROUP_SIZE waiters can enter concurrently and thus
885+ increment __wseq. Therefore, if the xor operation observes a value of
886+ __wseq, then the value it applies the modification to later on can be
887+ derived (see below).
888+
889+ One benefit of this scheme is that this makes load operations
890+ obstruction-free because unlike if we would just lock the counter, readers
891+ can almost always interpret a snapshot of each halves. Readers can be
892+ forced to read a new snapshot when the read is concurrent with an overflow.
893+ However, overflows will happen infrequently, so load operations are
894+ practically lock-free.
895+
896+ (*) The highest value we add is __PTHREAD_COND_MAX_GROUP_SIZE << 2 to
897+ __g1_start (the two extra bits are for the lock in the two LSBs of
898+ __g1_start). */
899+
900+typedef struct
901+{
902+ unsigned int low;
903+ unsigned int high;
904+} _condvar_lohi;
905+
906+static uint64_t
907+__condvar_fetch_add_64_relaxed (_condvar_lohi *lh, unsigned int op)
908+{
909+ /* S1. Note that this is an atomic read-modify-write so it extends the
910+ release sequence of release MO store at S3. */
911+ unsigned int l = atomic_fetch_add_relaxed (&lh->low, op);
912+ unsigned int h = atomic_load_relaxed (&lh->high);
913+ uint64_t result = ((uint64_t) h << 31) | l;
914+ l += op;
915+ if ((l >> 31) > 0)
916+ {
917+ /* Overflow. Need to increment higher-order half. Note that all
918+ add operations are ordered in happens-before. */
919+ h++;
920+ /* S2. Release MO to synchronize with the loads of the higher-order half
921+ in the load operation. See __condvar_load_64_relaxed. */
922+ atomic_store_release (&lh->high, h | ((unsigned int) 1 << 31));
923+ l ^= (unsigned int) 1 << 31;
924+ /* S3. See __condvar_load_64_relaxed. */
925+ atomic_store_release (&lh->low, l);
926+ /* S4. Likewise. */
927+ atomic_store_release (&lh->high, h);
928+ }
929+ return result;
930+}
931+
932+static uint64_t
933+__condvar_load_64_relaxed (_condvar_lohi *lh)
934+{
935+ unsigned int h, l, h2;
936+ do
937+ {
938+ /* This load and the second one below to the same location read from the
939+ stores in the overflow handling of the add operation or the
940+ initializing stores (which is a simple special case because
941+ initialization always completely happens before further use).
942+ Because no two stores to the higher-order half write the same value,
943+ the loop ensures that if we continue to use the snapshot, this load
944+ and the second one read from the same store operation. All candidate
945+ store operations have release MO.
946+ If we read from S2 in the first load, then we will see the value of
947+ S1 on the next load (because we synchronize with S2), or a value
948+ later in modification order. We correctly ignore the lower-half's
949+ overflow bit in this case. If we read from S4, then we will see the
950+ value of S3 in the next load (or a later value), which does not have
951+ the overflow bit set anymore.
952+ */
953+ h = atomic_load_acquire (&lh->high);
954+ /* This will read from the release sequence of S3 (i.e, either the S3
955+ store or the read-modify-writes at S1 following S3 in modification
956+ order). Thus, the read synchronizes with S3, and the following load
957+ of the higher-order half will read from the matching S2 (or a later
958+ value).
959+ Thus, if we read a lower-half value here that already overflowed and
960+ belongs to an increased higher-order half value, we will see the
961+ latter and h and h2 will not be equal. */
962+ l = atomic_load_acquire (&lh->low);
963+ /* See above. */
964+ h2 = atomic_load_relaxed (&lh->high);
965+ }
966+ while (h != h2);
967+ if (((l >> 31) > 0) && ((h >> 31) > 0))
968+ l ^= (unsigned int) 1 << 31;
969+ return ((uint64_t) (h & ~((unsigned int) 1 << 31)) << 31) + l;
970+}
971+
972+static uint64_t __attribute__ ((unused))
973+__condvar_load_wseq_relaxed (pthread_cond_t *cond)
974+{
975+ return __condvar_load_64_relaxed ((_condvar_lohi *) &cond->__data.__wseq32);
976+}
977+
978+static uint64_t __attribute__ ((unused))
979+__condvar_fetch_add_wseq_acquire (pthread_cond_t *cond, unsigned int val)
980+{
981+ uint64_t r = __condvar_fetch_add_64_relaxed
982+ ((_condvar_lohi *) &cond->__data.__wseq32, val);
983+ atomic_thread_fence_acquire ();
984+ return r;
985+}
986+
987+static uint64_t __attribute__ ((unused))
988+__condvar_fetch_xor_wseq_release (pthread_cond_t *cond, unsigned int val)
989+{
990+ _condvar_lohi *lh = (_condvar_lohi *) &cond->__data.__wseq32;
991+ /* First, get the current value. See __condvar_load_64_relaxed. */
992+ unsigned int h, l, h2;
993+ do
994+ {
995+ h = atomic_load_acquire (&lh->high);
996+ l = atomic_load_acquire (&lh->low);
997+ h2 = atomic_load_relaxed (&lh->high);
998+ }
999+ while (h != h2);
1000+ if (((l >> 31) > 0) && ((h >> 31) == 0))
1001+ h++;
1002+ h &= ~((unsigned int) 1 << 31);
1003+ l &= ~((unsigned int) 1 << 31);
1004+
1005+ /* Now modify. Due to the coherence rules, the prior load will read a value
1006+ earlier in modification order than the following fetch-xor.
1007+ This uses release MO to make the full operation have release semantics
1008+ (all other operations access the lower-order half). */
1009+ unsigned int l2 = atomic_fetch_xor_release (&lh->low, val)
1010+ & ~((unsigned int) 1 << 31);
1011+ if (l2 < l)
1012+ /* The lower-order half overflowed in the meantime. This happened exactly
1013+ once due to the limit on concurrent waiters (see above). */
1014+ h++;
1015+ return ((uint64_t) h << 31) + l2;
1016+}
1017+
1018+static uint64_t __attribute__ ((unused))
1019+__condvar_load_g1_start_relaxed (pthread_cond_t *cond)
1020+{
1021+ return __condvar_load_64_relaxed
1022+ ((_condvar_lohi *) &cond->__data.__g1_start32);
1023+}
1024+
1025+static void __attribute__ ((unused))
1026+__condvar_add_g1_start_relaxed (pthread_cond_t *cond, unsigned int val)
1027+{
1028+ ignore_value (__condvar_fetch_add_64_relaxed
1029+ ((_condvar_lohi *) &cond->__data.__g1_start32, val));
1030+}
1031+
1032+#endif /* !__HAVE_64B_ATOMICS */
1033+
1034+
1035+/* The lock that signalers use. See pthread_cond_wait_common for uses.
1036+ The lock is our normal three-state lock: not acquired (0) / acquired (1) /
1037+ acquired-with-futex_wake-request (2). However, we need to preserve the
1038+ other bits in the unsigned int used for the lock, and therefore it is a
1039+ little more complex. */
1040+static void __attribute__ ((unused))
1041+__condvar_acquire_lock (pthread_cond_t *cond, int private)
1042+{
1043+ unsigned int s = atomic_load_relaxed (&cond->__data.__g1_orig_size);
1044+ while ((s & 3) == 0)
1045+ {
1046+ if (atomic_compare_exchange_weak_acquire (&cond->__data.__g1_orig_size,
1047+ &s, s | 1))
1048+ return;
1049+ /* TODO Spinning and back-off. */
1050+ }
1051+ /* We can't change from not acquired to acquired, so try to change to
1052+ acquired-with-futex-wake-request and do a futex wait if we cannot change
1053+ from not acquired. */
1054+ while (1)
1055+ {
1056+ while ((s & 3) != 2)
1057+ {
1058+ if (atomic_compare_exchange_weak_acquire
1059+ (&cond->__data.__g1_orig_size, &s, (s & ~(unsigned int) 3) | 2))
1060+ {
1061+ if ((s & 3) == 0)
1062+ return;
1063+ break;
1064+ }
1065+ /* TODO Back off. */
1066+ }
1067+ futex_wait_simple (&cond->__data.__g1_orig_size,
1068+ (s & ~(unsigned int) 3) | 2, private);
1069+ /* Reload so we see a recent value. */
1070+ s = atomic_load_relaxed (&cond->__data.__g1_orig_size);
1071+ }
1072+}
1073+
1074+/* See __condvar_acquire_lock. */
1075+static void __attribute__ ((unused))
1076+__condvar_release_lock (pthread_cond_t *cond, int private)
1077+{
1078+ if ((atomic_fetch_and_release (&cond->__data.__g1_orig_size,
1079+ ~(unsigned int) 3) & 3)
1080+ == 2)
1081+ futex_wake (&cond->__data.__g1_orig_size, 1, private);
1082+}
1083+
1084+/* Only use this when having acquired the lock. */
1085+static unsigned int __attribute__ ((unused))
1086+__condvar_get_orig_size (pthread_cond_t *cond)
1087+{
1088+ return atomic_load_relaxed (&cond->__data.__g1_orig_size) >> 2;
1089+}
1090+
1091+/* Only use this when having acquired the lock. */
1092+static void __attribute__ ((unused))
1093+__condvar_set_orig_size (pthread_cond_t *cond, unsigned int size)
1094+{
1095+ /* We have acquired the lock, but might get one concurrent update due to a
1096+ lock state change from acquired to acquired-with-futex_wake-request.
1097+ The store with relaxed MO is fine because there will be no further
1098+ changes to the lock bits nor the size, and we will subsequently release
1099+ the lock with release MO. */
1100+ unsigned int s;
1101+ s = (atomic_load_relaxed (&cond->__data.__g1_orig_size) & 3)
1102+ | (size << 2);
1103+ if ((atomic_exchange_relaxed (&cond->__data.__g1_orig_size, s) & 3)
1104+ != (s & 3))
1105+ atomic_store_relaxed (&cond->__data.__g1_orig_size, (size << 2) | 2);
1106+}
1107+
1108+/* Returns FUTEX_SHARED or FUTEX_PRIVATE based on the provided __wrefs
1109+ value. */
1110+static int __attribute__ ((unused))
1111+__condvar_get_private (int flags)
1112+{
1113+ if ((flags & __PTHREAD_COND_SHARED_MASK) == 0)
1114+ return FUTEX_PRIVATE;
1115+ else
1116+ return FUTEX_SHARED;
1117+}
1118+
1119+/* This closes G1 (whose index is in G1INDEX), waits for all futex waiters to
1120+ leave G1, converts G1 into a fresh G2, and then switches group roles so that
1121+ the former G2 becomes the new G1 ending at the current __wseq value when we
1122+ eventually make the switch (WSEQ is just an observation of __wseq by the
1123+ signaler).
1124+ If G2 is empty, it will not switch groups because then it would create an
1125+ empty G1 which would require switching groups again on the next signal.
1126+ Returns false iff groups were not switched because G2 was empty. */
1127+static bool __attribute__ ((unused))
1128+__condvar_quiesce_and_switch_g1 (pthread_cond_t *cond, uint64_t wseq,
1129+ unsigned int *g1index, int private)
1130+{
1131+ const unsigned int maxspin = 0;
1132+ unsigned int g1 = *g1index;
1133+
1134+ /* If there is no waiter in G2, we don't do anything. The expression may
1135+ look odd but remember that __g_size might hold a negative value, so
1136+ putting the expression this way avoids relying on implementation-defined
1137+ behavior.
1138+ Note that this works correctly for a zero-initialized condvar too. */
1139+ unsigned int old_orig_size = __condvar_get_orig_size (cond);
1140+ uint64_t old_g1_start = __condvar_load_g1_start_relaxed (cond) >> 1;
1141+ if (((unsigned) (wseq - old_g1_start - old_orig_size)
1142+ + cond->__data.__g_size[g1 ^ 1]) == 0)
1143+ return false;
1144+
1145+ /* Now try to close and quiesce G1. We have to consider the following kinds
1146+ of waiters:
1147+ * Waiters from less recent groups than G1 are not affected because
1148+ nothing will change for them apart from __g1_start getting larger.
1149+ * New waiters arriving concurrently with the group switching will all go
1150+ into G2 until we atomically make the switch. Waiters existing in G2
1151+ are not affected.
1152+ * Waiters in G1 will be closed out immediately by setting a flag in
1153+ __g_signals, which will prevent waiters from blocking using a futex on
1154+ __g_signals and also notifies them that the group is closed. As a
1155+ result, they will eventually remove their group reference, allowing us
1156+ to close switch group roles. */
1157+
1158+ /* First, set the closed flag on __g_signals. This tells waiters that are
1159+ about to wait that they shouldn't do that anymore. This basically
1160+ serves as an advance notificaton of the upcoming change to __g1_start;
1161+ waiters interpret it as if __g1_start was larger than their waiter
1162+ sequence position. This allows us to change __g1_start after waiting
1163+ for all existing waiters with group references to leave, which in turn
1164+ makes recovery after stealing a signal simpler because it then can be
1165+ skipped if __g1_start indicates that the group is closed (otherwise,
1166+ we would have to recover always because waiters don't know how big their
1167+ groups are). Relaxed MO is fine. */
1168+ atomic_fetch_or_relaxed (cond->__data.__g_signals + g1, 1);
1169+
1170+ /* Wait until there are no group references anymore. The fetch-or operation
1171+ injects us into the modification order of __g_refs; release MO ensures
1172+ that waiters incrementing __g_refs after our fetch-or see the previous
1173+ changes to __g_signals and to __g1_start that had to happen before we can
1174+ switch this G1 and alias with an older group (we have two groups, so
1175+ aliasing requires switching group roles twice). Note that nobody else
1176+ can have set the wake-request flag, so we do not have to act upon it.
1177+
1178+ Also note that it is harmless if older waiters or waiters from this G1
1179+ get a group reference after we have quiesced the group because it will
1180+ remain closed for them either because of the closed flag in __g_signals
1181+ or the later update to __g1_start. New waiters will never arrive here
1182+ but instead continue to go into the still current G2. */
1183+ unsigned r = atomic_fetch_or_release (cond->__data.__g_refs + g1, 0);
1184+ while ((r >> 1) > 0)
1185+ {
1186+ for (unsigned int spin = maxspin; ((r >> 1) > 0) && (spin > 0); spin--)
1187+ {
1188+ /* TODO Back off. */
1189+ r = atomic_load_relaxed (cond->__data.__g_refs + g1);
1190+ }
1191+ if ((r >> 1) > 0)
1192+ {
1193+ /* There is still a waiter after spinning. Set the wake-request
1194+ flag and block. Relaxed MO is fine because this is just about
1195+ this futex word. */
1196+ r = atomic_fetch_or_relaxed (cond->__data.__g_refs + g1, 1);
1197+
1198+ if ((r >> 1) > 0)
1199+ futex_wait_simple (cond->__data.__g_refs + g1, r, private);
1200+ /* Reload here so we eventually see the most recent value even if we
1201+ do not spin. */
1202+ r = atomic_load_relaxed (cond->__data.__g_refs + g1);
1203+ }
1204+ }
1205+ /* Acquire MO so that we synchronize with the release operation that waiters
1206+ use to decrement __g_refs and thus happen after the waiters we waited
1207+ for. */
1208+ atomic_thread_fence_acquire ();
1209+
1210+ /* Update __g1_start, which finishes closing this group. The value we add
1211+ will never be negative because old_orig_size can only be zero when we
1212+ switch groups the first time after a condvar was initialized, in which
1213+ case G1 will be at index 1 and we will add a value of 1. See above for
1214+ why this takes place after waiting for quiescence of the group.
1215+ Relaxed MO is fine because the change comes with no additional
1216+ constraints that others would have to observe. */
1217+ __condvar_add_g1_start_relaxed (cond,
1218+ (old_orig_size << 1) + (g1 == 1 ? 1 : - 1));
1219+
1220+ /* Now reopen the group, thus enabling waiters to again block using the
1221+ futex controlled by __g_signals. Release MO so that observers that see
1222+ no signals (and thus can block) also see the write __g1_start and thus
1223+ that this is now a new group (see __pthread_cond_wait_common for the
1224+ matching acquire MO loads). */
1225+ atomic_store_release (cond->__data.__g_signals + g1, 0);
1226+
1227+ /* At this point, the old G1 is now a valid new G2 (but not in use yet).
1228+ No old waiter can neither grab a signal nor acquire a reference without
1229+ noticing that __g1_start is larger.
1230+ We can now publish the group switch by flipping the G2 index in __wseq.
1231+ Release MO so that this synchronizes with the acquire MO operation
1232+ waiters use to obtain a position in the waiter sequence. */
1233+ wseq = __condvar_fetch_xor_wseq_release (cond, 1) >> 1;
1234+ g1 ^= 1;
1235+ *g1index ^= 1;
1236+
1237+ /* These values are just observed by signalers, and thus protected by the
1238+ lock. */
1239+ unsigned int orig_size = wseq - (old_g1_start + old_orig_size);
1240+ __condvar_set_orig_size (cond, orig_size);
1241+ /* Use and addition to not loose track of cancellations in what was
1242+ previously G2. */
1243+ cond->__data.__g_size[g1] += orig_size;
1244+
1245+ /* The new G1's size may be zero because of cancellations during its time
1246+ as G2. If this happens, there are no waiters that have to receive a
1247+ signal, so we do not need to add any and return false. */
1248+ if (cond->__data.__g_size[g1] == 0)
1249+ return false;
1250+
1251+ return true;
1252+}
1253diff --git a/nptl/pthread_cond_destroy.c b/nptl/pthread_cond_destroy.c
1254index 1acd804..5845c6a 100644
1255--- a/nptl/pthread_cond_destroy.c
1256+++ b/nptl/pthread_cond_destroy.c
1257@@ -20,66 +20,42 @@
1258 #include <shlib-compat.h>
1259 #include "pthreadP.h"
1260 #include <stap-probe.h>
1261-
1262-
1263+#include <atomic.h>
1264+#include <futex-internal.h>
1265+
1266+#include "pthread_cond_common.c"
1267+
1268+/* See __pthread_cond_wait for a high-level description of the algorithm.
1269+
1270+ A correct program must make sure that no waiters are blocked on the condvar
1271+ when it is destroyed, and that there are no concurrent signals or
1272+ broadcasts. To wake waiters reliably, the program must signal or
1273+ broadcast while holding the mutex or after having held the mutex. It must
1274+ also ensure that no signal or broadcast are still pending to unblock
1275+ waiters; IOW, because waiters can wake up spuriously, the program must
1276+ effectively ensure that destruction happens after the execution of those
1277+ signal or broadcast calls.
1278+ Thus, we can assume that all waiters that are still accessing the condvar
1279+ have been woken. We wait until they have confirmed to have woken up by
1280+ decrementing __wrefs. */
1281 int
1282 __pthread_cond_destroy (pthread_cond_t *cond)
1283 {
1284- int pshared = (cond->__data.__mutex == (void *) ~0l)
1285- ? LLL_SHARED : LLL_PRIVATE;
1286-
1287 LIBC_PROBE (cond_destroy, 1, cond);
1288
1289- /* Make sure we are alone. */
1290- lll_lock (cond->__data.__lock, pshared);
1291-
1292- if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
1293- {
1294- /* If there are still some waiters which have not been
1295- woken up, this is an application bug. */
1296- lll_unlock (cond->__data.__lock, pshared);
1297- return EBUSY;
1298- }
1299-
1300- /* Tell pthread_cond_*wait that this condvar is being destroyed. */
1301- cond->__data.__total_seq = -1ULL;
1302-
1303- /* If there are waiters which have been already signalled or
1304- broadcasted, but still are using the pthread_cond_t structure,
1305- pthread_cond_destroy needs to wait for them. */
1306- unsigned int nwaiters = cond->__data.__nwaiters;
1307-
1308- if (nwaiters >= (1 << COND_NWAITERS_SHIFT))
1309+ /* Set the wake request flag. We could also spin, but destruction that is
1310+ concurrent with still-active waiters is probably neither common nor
1311+ performance critical. Acquire MO to synchronize with waiters confirming
1312+ that they finished. */
1313+ unsigned int wrefs = atomic_fetch_or_acquire (&cond->__data.__wrefs, 4);
1314+ int private = __condvar_get_private (wrefs);
1315+ while (wrefs >> 3 != 0)
1316 {
1317- /* Wake everybody on the associated mutex in case there are
1318- threads that have been requeued to it.
1319- Without this, pthread_cond_destroy could block potentially
1320- for a long time or forever, as it would depend on other
1321- thread's using the mutex.
1322- When all threads waiting on the mutex are woken up, pthread_cond_wait
1323- only waits for threads to acquire and release the internal
1324- condvar lock. */
1325- if (cond->__data.__mutex != NULL
1326- && cond->__data.__mutex != (void *) ~0l)
1327- {
1328- pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
1329- lll_futex_wake (&mut->__data.__lock, INT_MAX,
1330- PTHREAD_MUTEX_PSHARED (mut));
1331- }
1332-
1333- do
1334- {
1335- lll_unlock (cond->__data.__lock, pshared);
1336-
1337- lll_futex_wait (&cond->__data.__nwaiters, nwaiters, pshared);
1338-
1339- lll_lock (cond->__data.__lock, pshared);
1340-
1341- nwaiters = cond->__data.__nwaiters;
1342- }
1343- while (nwaiters >= (1 << COND_NWAITERS_SHIFT));
1344+ futex_wait_simple (&cond->__data.__wrefs, wrefs, private);
1345+ /* See above. */
1346+ wrefs = atomic_load_acquire (&cond->__data.__wrefs);
1347 }
1348-
1349+ /* The memory the condvar occupies can now be reused. */
1350 return 0;
1351 }
1352 versioned_symbol (libpthread, __pthread_cond_destroy,
1353diff --git a/nptl/pthread_cond_init.c b/nptl/pthread_cond_init.c
1354index 9023370..c1eac5f 100644
1355--- a/nptl/pthread_cond_init.c
1356+++ b/nptl/pthread_cond_init.c
1357@@ -19,25 +19,29 @@
1358 #include <shlib-compat.h>
1359 #include "pthreadP.h"
1360 #include <stap-probe.h>
1361+#include <string.h>
1362
1363
1364+/* See __pthread_cond_wait for details. */
1365 int
1366 __pthread_cond_init (pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
1367 {
1368 struct pthread_condattr *icond_attr = (struct pthread_condattr *) cond_attr;
1369
1370- cond->__data.__lock = LLL_LOCK_INITIALIZER;
1371- cond->__data.__futex = 0;
1372- cond->__data.__nwaiters = (icond_attr != NULL
1373- ? ((icond_attr->value >> 1)
1374- & ((1 << COND_NWAITERS_SHIFT) - 1))
1375- : CLOCK_REALTIME);
1376- cond->__data.__total_seq = 0;
1377- cond->__data.__wakeup_seq = 0;
1378- cond->__data.__woken_seq = 0;
1379- cond->__data.__mutex = (icond_attr == NULL || (icond_attr->value & 1) == 0
1380- ? NULL : (void *) ~0l);
1381- cond->__data.__broadcast_seq = 0;
1382+ memset (cond, 0, sizeof (pthread_cond_t));
1383+
1384+ /* Update the pretty printers if the internal representation of icond_attr
1385+ is changed. */
1386+
1387+ /* Iff not equal to ~0l, this is a PTHREAD_PROCESS_PRIVATE condvar. */
1388+ if (icond_attr != NULL && (icond_attr->value & 1) != 0)
1389+ cond->__data.__wrefs |= __PTHREAD_COND_SHARED_MASK;
1390+ int clockid = (icond_attr != NULL
1391+ ? ((icond_attr->value >> 1) & ((1 << COND_CLOCK_BITS) - 1))
1392+ : CLOCK_REALTIME);
1393+ /* If 0, CLOCK_REALTIME is used; CLOCK_MONOTONIC otherwise. */
1394+ if (clockid != CLOCK_REALTIME)
1395+ cond->__data.__wrefs |= __PTHREAD_COND_CLOCK_MONOTONIC_MASK;
1396
1397 LIBC_PROBE (cond_init, 2, cond, cond_attr);
1398
1399diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c
1400index b3a6d3d..a95d569 100644
1401--- a/nptl/pthread_cond_signal.c
1402+++ b/nptl/pthread_cond_signal.c
1403@@ -19,62 +19,79 @@
1404 #include <endian.h>
1405 #include <errno.h>
1406 #include <sysdep.h>
1407-#include <lowlevellock.h>
1408+#include <futex-internal.h>
1409 #include <pthread.h>
1410 #include <pthreadP.h>
1411+#include <atomic.h>
1412+#include <stdint.h>
1413
1414 #include <shlib-compat.h>
1415-#include <kernel-features.h>
1416 #include <stap-probe.h>
1417
1418+#include "pthread_cond_common.c"
1419
1420+/* See __pthread_cond_wait for a high-level description of the algorithm. */
1421 int
1422 __pthread_cond_signal (pthread_cond_t *cond)
1423 {
1424- int pshared = (cond->__data.__mutex == (void *) ~0l)
1425- ? LLL_SHARED : LLL_PRIVATE;
1426-
1427 LIBC_PROBE (cond_signal, 1, cond);
1428
1429- /* Make sure we are alone. */
1430- lll_lock (cond->__data.__lock, pshared);
1431-
1432- /* Are there any waiters to be woken? */
1433- if (cond->__data.__total_seq > cond->__data.__wakeup_seq)
1434+ /* First check whether there are waiters. Relaxed MO is fine for that for
1435+ the same reasons that relaxed MO is fine when observing __wseq (see
1436+ below). */
1437+ unsigned int wrefs = atomic_load_relaxed (&cond->__data.__wrefs);
1438+ if (wrefs >> 3 == 0)
1439+ return 0;
1440+ int private = __condvar_get_private (wrefs);
1441+
1442+ __condvar_acquire_lock (cond, private);
1443+
1444+ /* Load the waiter sequence number, which represents our relative ordering
1445+ to any waiters. Relaxed MO is sufficient for that because:
1446+ 1) We can pick any position that is allowed by external happens-before
1447+ constraints. In particular, if another __pthread_cond_wait call
1448+ happened before us, this waiter must be eligible for being woken by
1449+ us. The only way do establish such a happens-before is by signaling
1450+ while having acquired the mutex associated with the condvar and
1451+ ensuring that the signal's critical section happens after the waiter.
1452+ Thus, the mutex ensures that we see that waiter's __wseq increase.
1453+ 2) Once we pick a position, we do not need to communicate this to the
1454+ program via a happens-before that we set up: First, any wake-up could
1455+ be a spurious wake-up, so the program must not interpret a wake-up as
1456+ an indication that the waiter happened before a particular signal;
1457+ second, a program cannot detect whether a waiter has not yet been
1458+ woken (i.e., it cannot distinguish between a non-woken waiter and one
1459+ that has been woken but hasn't resumed execution yet), and thus it
1460+ cannot try to deduce that a signal happened before a particular
1461+ waiter. */
1462+ unsigned long long int wseq = __condvar_load_wseq_relaxed (cond);
1463+ unsigned int g1 = (wseq & 1) ^ 1;
1464+ wseq >>= 1;
1465+ bool do_futex_wake = false;
1466+
1467+ /* If G1 is still receiving signals, we put the signal there. If not, we
1468+ check if G2 has waiters, and if so, quiesce and switch G1 to the former
1469+ G2; if this results in a new G1 with waiters (G2 might have cancellations
1470+ already, see __condvar_quiesce_and_switch_g1), we put the signal in the
1471+ new G1. */
1472+ if ((cond->__data.__g_size[g1] != 0)
1473+ || __condvar_quiesce_and_switch_g1 (cond, wseq, &g1, private))
1474 {
1475- /* Yes. Mark one of them as woken. */
1476- ++cond->__data.__wakeup_seq;
1477- ++cond->__data.__futex;
1478-
1479-#if (defined lll_futex_cmp_requeue_pi \
1480- && defined __ASSUME_REQUEUE_PI)
1481- pthread_mutex_t *mut = cond->__data.__mutex;
1482-
1483- if (USE_REQUEUE_PI (mut)
1484- /* This can only really fail with a ENOSYS, since nobody can modify
1485- futex while we have the cond_lock. */
1486- && lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,
1487- &mut->__data.__lock,
1488- cond->__data.__futex, pshared) == 0)
1489- {
1490- lll_unlock (cond->__data.__lock, pshared);
1491- return 0;
1492- }
1493- else
1494-#endif
1495- /* Wake one. */
1496- if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
1497- 1, 1,
1498- &cond->__data.__lock,
1499- pshared), 0))
1500- return 0;
1501-
1502- /* Fallback if neither of them work. */
1503- lll_futex_wake (&cond->__data.__futex, 1, pshared);
1504+ /* Add a signal. Relaxed MO is fine because signaling does not need to
1505+ establish a happens-before relation (see above). We do not mask the
1506+ release-MO store when initializing a group in
1507+ __condvar_quiesce_and_switch_g1 because we use an atomic
1508+ read-modify-write and thus extend that store's release sequence. */
1509+ atomic_fetch_add_relaxed (cond->__data.__g_signals + g1, 2);
1510+ cond->__data.__g_size[g1]--;
1511+ /* TODO Only set it if there are indeed futex waiters. */
1512+ do_futex_wake = true;
1513 }
1514
1515- /* We are done. */
1516- lll_unlock (cond->__data.__lock, pshared);
1517+ __condvar_release_lock (cond, private);
1518+
1519+ if (do_futex_wake)
1520+ futex_wake (cond->__data.__g_signals + g1, 1, private);
1521
1522 return 0;
1523 }
1524diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c
1525deleted file mode 100644
1526index 711a51d..0000000
1527--- a/nptl/pthread_cond_timedwait.c
1528+++ /dev/null
1529@@ -1,268 +0,0 @@
1530-/* Copyright (C) 2003-2016 Free Software Foundation, Inc.
1531- This file is part of the GNU C Library.
1532- Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003.
1533-
1534- The GNU C Library is free software; you can redistribute it and/or
1535- modify it under the terms of the GNU Lesser General Public
1536- License as published by the Free Software Foundation; either
1537- version 2.1 of the License, or (at your option) any later version.
1538-
1539- The GNU C Library is distributed in the hope that it will be useful,
1540- but WITHOUT ANY WARRANTY; without even the implied warranty of
1541- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1542- Lesser General Public License for more details.
1543-
1544- You should have received a copy of the GNU Lesser General Public
1545- License along with the GNU C Library; if not, see
1546- <http://www.gnu.org/licenses/>. */
1547-
1548-#include <endian.h>
1549-#include <errno.h>
1550-#include <sysdep.h>
1551-#include <lowlevellock.h>
1552-#include <pthread.h>
1553-#include <pthreadP.h>
1554-#include <sys/time.h>
1555-#include <kernel-features.h>
1556-
1557-#include <shlib-compat.h>
1558-
1559-#ifndef HAVE_CLOCK_GETTIME_VSYSCALL
1560-# undef INTERNAL_VSYSCALL
1561-# define INTERNAL_VSYSCALL INTERNAL_SYSCALL
1562-# undef INLINE_VSYSCALL
1563-# define INLINE_VSYSCALL INLINE_SYSCALL
1564-#else
1565-# include <libc-vdso.h>
1566-#endif
1567-
1568-/* Cleanup handler, defined in pthread_cond_wait.c. */
1569-extern void __condvar_cleanup (void *arg)
1570- __attribute__ ((visibility ("hidden")));
1571-
1572-struct _condvar_cleanup_buffer
1573-{
1574- int oldtype;
1575- pthread_cond_t *cond;
1576- pthread_mutex_t *mutex;
1577- unsigned int bc_seq;
1578-};
1579-
1580-int
1581-__pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
1582- const struct timespec *abstime)
1583-{
1584- struct _pthread_cleanup_buffer buffer;
1585- struct _condvar_cleanup_buffer cbuffer;
1586- int result = 0;
1587-
1588- /* Catch invalid parameters. */
1589- if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
1590- return EINVAL;
1591-
1592- int pshared = (cond->__data.__mutex == (void *) ~0l)
1593- ? LLL_SHARED : LLL_PRIVATE;
1594-
1595-#if (defined lll_futex_timed_wait_requeue_pi \
1596- && defined __ASSUME_REQUEUE_PI)
1597- int pi_flag = 0;
1598-#endif
1599-
1600- /* Make sure we are alone. */
1601- lll_lock (cond->__data.__lock, pshared);
1602-
1603- /* Now we can release the mutex. */
1604- int err = __pthread_mutex_unlock_usercnt (mutex, 0);
1605- if (err)
1606- {
1607- lll_unlock (cond->__data.__lock, pshared);
1608- return err;
1609- }
1610-
1611- /* We have one new user of the condvar. */
1612- ++cond->__data.__total_seq;
1613- ++cond->__data.__futex;
1614- cond->__data.__nwaiters += 1 << COND_NWAITERS_SHIFT;
1615-
1616- /* Work around the fact that the kernel rejects negative timeout values
1617- despite them being valid. */
1618- if (__glibc_unlikely (abstime->tv_sec < 0))
1619- goto timeout;
1620-
1621- /* Remember the mutex we are using here. If there is already a
1622- different address store this is a bad user bug. Do not store
1623- anything for pshared condvars. */
1624- if (cond->__data.__mutex != (void *) ~0l)
1625- cond->__data.__mutex = mutex;
1626-
1627- /* Prepare structure passed to cancellation handler. */
1628- cbuffer.cond = cond;
1629- cbuffer.mutex = mutex;
1630-
1631- /* Before we block we enable cancellation. Therefore we have to
1632- install a cancellation handler. */
1633- __pthread_cleanup_push (&buffer, __condvar_cleanup, &cbuffer);
1634-
1635- /* The current values of the wakeup counter. The "woken" counter
1636- must exceed this value. */
1637- unsigned long long int val;
1638- unsigned long long int seq;
1639- val = seq = cond->__data.__wakeup_seq;
1640- /* Remember the broadcast counter. */
1641- cbuffer.bc_seq = cond->__data.__broadcast_seq;
1642-
1643- while (1)
1644- {
1645-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
1646- || !defined lll_futex_timed_wait_bitset)
1647- struct timespec rt;
1648- {
1649-# ifdef __NR_clock_gettime
1650- INTERNAL_SYSCALL_DECL (err);
1651- (void) INTERNAL_VSYSCALL (clock_gettime, err, 2,
1652- (cond->__data.__nwaiters
1653- & ((1 << COND_NWAITERS_SHIFT) - 1)),
1654- &rt);
1655- /* Convert the absolute timeout value to a relative timeout. */
1656- rt.tv_sec = abstime->tv_sec - rt.tv_sec;
1657- rt.tv_nsec = abstime->tv_nsec - rt.tv_nsec;
1658-# else
1659- /* Get the current time. So far we support only one clock. */
1660- struct timeval tv;
1661- (void) __gettimeofday (&tv, NULL);
1662-
1663- /* Convert the absolute timeout value to a relative timeout. */
1664- rt.tv_sec = abstime->tv_sec - tv.tv_sec;
1665- rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000;
1666-# endif
1667- }
1668- if (rt.tv_nsec < 0)
1669- {
1670- rt.tv_nsec += 1000000000;
1671- --rt.tv_sec;
1672- }
1673- /* Did we already time out? */
1674- if (__glibc_unlikely (rt.tv_sec < 0))
1675- {
1676- if (cbuffer.bc_seq != cond->__data.__broadcast_seq)
1677- goto bc_out;
1678-
1679- goto timeout;
1680- }
1681-#endif
1682-
1683- unsigned int futex_val = cond->__data.__futex;
1684-
1685- /* Prepare to wait. Release the condvar futex. */
1686- lll_unlock (cond->__data.__lock, pshared);
1687-
1688- /* Enable asynchronous cancellation. Required by the standard. */
1689- cbuffer.oldtype = __pthread_enable_asynccancel ();
1690-
1691-/* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient
1692- to check just the former. */
1693-#if (defined lll_futex_timed_wait_requeue_pi \
1694- && defined __ASSUME_REQUEUE_PI)
1695- /* If pi_flag remained 1 then it means that we had the lock and the mutex
1696- but a spurious waker raced ahead of us. Give back the mutex before
1697- going into wait again. */
1698- if (pi_flag)
1699- {
1700- __pthread_mutex_cond_lock_adjust (mutex);
1701- __pthread_mutex_unlock_usercnt (mutex, 0);
1702- }
1703- pi_flag = USE_REQUEUE_PI (mutex);
1704-
1705- if (pi_flag)
1706- {
1707- unsigned int clockbit = (cond->__data.__nwaiters & 1
1708- ? 0 : FUTEX_CLOCK_REALTIME);
1709- err = lll_futex_timed_wait_requeue_pi (&cond->__data.__futex,
1710- futex_val, abstime, clockbit,
1711- &mutex->__data.__lock,
1712- pshared);
1713- pi_flag = (err == 0);
1714- }
1715- else
1716-#endif
1717-
1718- {
1719-#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
1720- || !defined lll_futex_timed_wait_bitset)
1721- /* Wait until woken by signal or broadcast. */
1722- err = lll_futex_timed_wait (&cond->__data.__futex,
1723- futex_val, &rt, pshared);
1724-#else
1725- unsigned int clockbit = (cond->__data.__nwaiters & 1
1726- ? 0 : FUTEX_CLOCK_REALTIME);
1727- err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
1728- abstime, clockbit, pshared);
1729-#endif
1730- }
1731-
1732- /* Disable asynchronous cancellation. */
1733- __pthread_disable_asynccancel (cbuffer.oldtype);
1734-
1735- /* We are going to look at shared data again, so get the lock. */
1736- lll_lock (cond->__data.__lock, pshared);
1737-
1738- /* If a broadcast happened, we are done. */
1739- if (cbuffer.bc_seq != cond->__data.__broadcast_seq)
1740- goto bc_out;
1741-
1742- /* Check whether we are eligible for wakeup. */
1743- val = cond->__data.__wakeup_seq;
1744- if (val != seq && cond->__data.__woken_seq != val)
1745- break;
1746-
1747- /* Not woken yet. Maybe the time expired? */
1748- if (__glibc_unlikely (err == -ETIMEDOUT))
1749- {
1750- timeout:
1751- /* Yep. Adjust the counters. */
1752- ++cond->__data.__wakeup_seq;
1753- ++cond->__data.__futex;
1754-
1755- /* The error value. */
1756- result = ETIMEDOUT;
1757- break;
1758- }
1759- }
1760-
1761- /* Another thread woken up. */
1762- ++cond->__data.__woken_seq;
1763-
1764- bc_out:
1765-
1766- cond->__data.__nwaiters -= 1 << COND_NWAITERS_SHIFT;
1767-
1768- /* If pthread_cond_destroy was called on this variable already,
1769- notify the pthread_cond_destroy caller all waiters have left
1770- and it can be successfully destroyed. */
1771- if (cond->__data.__total_seq == -1ULL
1772- && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
1773- lll_futex_wake (&cond->__data.__nwaiters, 1, pshared);
1774-
1775- /* We are done with the condvar. */
1776- lll_unlock (cond->__data.__lock, pshared);
1777-
1778- /* The cancellation handling is back to normal, remove the handler. */
1779- __pthread_cleanup_pop (&buffer, 0);
1780-
1781- /* Get the mutex before returning. */
1782-#if (defined lll_futex_timed_wait_requeue_pi \
1783- && defined __ASSUME_REQUEUE_PI)
1784- if (pi_flag)
1785- {
1786- __pthread_mutex_cond_lock_adjust (mutex);
1787- err = 0;
1788- }
1789- else
1790-#endif
1791- err = __pthread_mutex_cond_lock (mutex);
1792-
1793- return err ?: result;
1794-}
1795-
1796-versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
1797- GLIBC_2_3_2);
1798diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c
1799index 3f62acc..2b43402 100644
1800--- a/nptl/pthread_cond_wait.c
1801+++ b/nptl/pthread_cond_wait.c
1802@@ -19,219 +19,655 @@
1803 #include <endian.h>
1804 #include <errno.h>
1805 #include <sysdep.h>
1806-#include <lowlevellock.h>
1807+#include <futex-internal.h>
1808 #include <pthread.h>
1809 #include <pthreadP.h>
1810-#include <kernel-features.h>
1811+#include <sys/time.h>
1812+#include <atomic.h>
1813+#include <stdint.h>
1814+#include <stdbool.h>
1815
1816 #include <shlib-compat.h>
1817 #include <stap-probe.h>
1818+#include <time.h>
1819+
1820+#include "pthread_cond_common.c"
1821+
1822
1823 struct _condvar_cleanup_buffer
1824 {
1825- int oldtype;
1826+ uint64_t wseq;
1827 pthread_cond_t *cond;
1828 pthread_mutex_t *mutex;
1829- unsigned int bc_seq;
1830+ int private;
1831 };
1832
1833
1834-void
1835-__attribute__ ((visibility ("hidden")))
1836-__condvar_cleanup (void *arg)
1837+/* Decrease the waiter reference count. */
1838+static void
1839+__condvar_confirm_wakeup (pthread_cond_t *cond, int private)
1840 {
1841- struct _condvar_cleanup_buffer *cbuffer =
1842- (struct _condvar_cleanup_buffer *) arg;
1843- unsigned int destroying;
1844- int pshared = (cbuffer->cond->__data.__mutex == (void *) ~0l)
1845- ? LLL_SHARED : LLL_PRIVATE;
1846+ /* If destruction is pending (i.e., the wake-request flag is nonzero) and we
1847+ are the last waiter (prior value of __wrefs was 1 << 3), then wake any
1848+ threads waiting in pthread_cond_destroy. Release MO to synchronize with
1849+ these threads. Don't bother clearing the wake-up request flag. */
1850+ if ((atomic_fetch_add_release (&cond->__data.__wrefs, -8) >> 2) == 3)
1851+ futex_wake (&cond->__data.__wrefs, INT_MAX, private);
1852+}
1853+
1854
1855- /* We are going to modify shared data. */
1856- lll_lock (cbuffer->cond->__data.__lock, pshared);
1857+/* Cancel waiting after having registered as a waiter previously. SEQ is our
1858+ position and G is our group index.
1859+ The goal of cancellation is to make our group smaller if that is still
1860+ possible. If we are in a closed group, this is not possible anymore; in
1861+ this case, we need to send a replacement signal for the one we effectively
1862+ consumed because the signal should have gotten consumed by another waiter
1863+ instead; we must not both cancel waiting and consume a signal.
1864+
1865+ Must not be called while still holding a reference on the group.
1866+
1867+ Returns true iff we consumed a signal.
1868+
1869+ On some kind of timeouts, we may be able to pretend that a signal we
1870+ effectively consumed happened before the timeout (i.e., similarly to first
1871+ spinning on signals before actually checking whether the timeout has
1872+ passed already). Doing this would allow us to skip sending a replacement
1873+ signal, but this case might happen rarely because the end of the timeout
1874+ must race with someone else sending a signal. Therefore, we don't bother
1875+ trying to optimize this. */
1876+static void
1877+__condvar_cancel_waiting (pthread_cond_t *cond, uint64_t seq, unsigned int g,
1878+ int private)
1879+{
1880+ bool consumed_signal = false;
1881
1882- if (cbuffer->bc_seq == cbuffer->cond->__data.__broadcast_seq)
1883+ /* No deadlock with group switching is possible here because we have do
1884+ not hold a reference on the group. */
1885+ __condvar_acquire_lock (cond, private);
1886+
1887+ uint64_t g1_start = __condvar_load_g1_start_relaxed (cond) >> 1;
1888+ if (g1_start > seq)
1889+ {
1890+ /* Our group is closed, so someone provided enough signals for it.
1891+ Thus, we effectively consumed a signal. */
1892+ consumed_signal = true;
1893+ }
1894+ else
1895 {
1896- /* This thread is not waiting anymore. Adjust the sequence counters
1897- appropriately. We do not increment WAKEUP_SEQ if this would
1898- bump it over the value of TOTAL_SEQ. This can happen if a thread
1899- was woken and then canceled. */
1900- if (cbuffer->cond->__data.__wakeup_seq
1901- < cbuffer->cond->__data.__total_seq)
1902+ if (g1_start + __condvar_get_orig_size (cond) <= seq)
1903+ {
1904+ /* We are in the current G2 and thus cannot have consumed a signal.
1905+ Reduce its effective size or handle overflow. Remember that in
1906+ G2, unsigned int size is zero or a negative value. */
1907+ if (cond->__data.__g_size[g] + __PTHREAD_COND_MAX_GROUP_SIZE > 0)
1908+ {
1909+ cond->__data.__g_size[g]--;
1910+ }
1911+ else
1912+ {
1913+ /* Cancellations would overflow the maximum group size. Just
1914+ wake up everyone spuriously to create a clean state. This
1915+ also means we do not consume a signal someone else sent. */
1916+ __condvar_release_lock (cond, private);
1917+ __pthread_cond_broadcast (cond);
1918+ return;
1919+ }
1920+ }
1921+ else
1922 {
1923- ++cbuffer->cond->__data.__wakeup_seq;
1924- ++cbuffer->cond->__data.__futex;
1925+ /* We are in current G1. If the group's size is zero, someone put
1926+ a signal in the group that nobody else but us can consume. */
1927+ if (cond->__data.__g_size[g] == 0)
1928+ consumed_signal = true;
1929+ else
1930+ {
1931+ /* Otherwise, we decrease the size of the group. This is
1932+ equivalent to atomically putting in a signal just for us and
1933+ consuming it right away. We do not consume a signal sent
1934+ by someone else. We also cannot have consumed a futex
1935+ wake-up because if we were cancelled or timed out in a futex
1936+ call, the futex will wake another waiter. */
1937+ cond->__data.__g_size[g]--;
1938+ }
1939 }
1940- ++cbuffer->cond->__data.__woken_seq;
1941 }
1942
1943- cbuffer->cond->__data.__nwaiters -= 1 << COND_NWAITERS_SHIFT;
1944+ __condvar_release_lock (cond, private);
1945
1946- /* If pthread_cond_destroy was called on this variable already,
1947- notify the pthread_cond_destroy caller all waiters have left
1948- and it can be successfully destroyed. */
1949- destroying = 0;
1950- if (cbuffer->cond->__data.__total_seq == -1ULL
1951- && cbuffer->cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
1952+ if (consumed_signal)
1953 {
1954- lll_futex_wake (&cbuffer->cond->__data.__nwaiters, 1, pshared);
1955- destroying = 1;
1956+ /* We effectively consumed a signal even though we didn't want to.
1957+ Therefore, we need to send a replacement signal.
1958+ If we would want to optimize this, we could do what
1959+ pthread_cond_signal does right in the critical section above. */
1960+ __pthread_cond_signal (cond);
1961 }
1962+}
1963
1964- /* We are done. */
1965- lll_unlock (cbuffer->cond->__data.__lock, pshared);
1966-
1967- /* Wake everybody to make sure no condvar signal gets lost. */
1968- if (! destroying)
1969- lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared);
1970-
1971- /* Get the mutex before returning unless asynchronous cancellation
1972- is in effect. We don't try to get the mutex if we already own it. */
1973- if (!(USE_REQUEUE_PI (cbuffer->mutex))
1974- || ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK)
1975- != THREAD_GETMEM (THREAD_SELF, tid)))
1976- {
1977- __pthread_mutex_cond_lock (cbuffer->mutex);
1978- }
1979- else
1980- __pthread_mutex_cond_lock_adjust (cbuffer->mutex);
1981+/* Wake up any signalers that might be waiting. */
1982+static void
1983+__condvar_dec_grefs (pthread_cond_t *cond, unsigned int g, int private)
1984+{
1985+ /* Release MO to synchronize-with the acquire load in
1986+ __condvar_quiesce_and_switch_g1. */
1987+ if (atomic_fetch_add_release (cond->__data.__g_refs + g, -2) == 3)
1988+ {
1989+ /* Clear the wake-up request flag before waking up. We do not need more
1990+ than relaxed MO and it doesn't matter if we apply this for an aliased
1991+ group because we wake all futex waiters right after clearing the
1992+ flag. */
1993+ atomic_fetch_and_relaxed (cond->__data.__g_refs + g, ~(unsigned int) 1);
1994+ futex_wake (cond->__data.__g_refs + g, INT_MAX, private);
1995+ }
1996 }
1997
1998+/* Clean-up for cancellation of waiters waiting for normal signals. We cancel
1999+ our registration as a waiter, confirm we have woken up, and re-acquire the
2000+ mutex. */
2001+static void
2002+__condvar_cleanup_waiting (void *arg)
2003+{
2004+ struct _condvar_cleanup_buffer *cbuffer =
2005+ (struct _condvar_cleanup_buffer *) arg;
2006+ pthread_cond_t *cond = cbuffer->cond;
2007+ unsigned g = cbuffer->wseq & 1;
2008
2009-int
2010-__pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
2011+ __condvar_dec_grefs (cond, g, cbuffer->private);
2012+
2013+ __condvar_cancel_waiting (cond, cbuffer->wseq >> 1, g, cbuffer->private);
2014+ /* FIXME With the current cancellation implementation, it is possible that
2015+ a thread is cancelled after it has returned from a syscall. This could
2016+ result in a cancelled waiter consuming a futex wake-up that is then
2017+ causing another waiter in the same group to not wake up. To work around
2018+ this issue until we have fixed cancellation, just add a futex wake-up
2019+ conservatively. */
2020+ futex_wake (cond->__data.__g_signals + g, 1, cbuffer->private);
2021+
2022+ __condvar_confirm_wakeup (cond, cbuffer->private);
2023+
2024+ /* XXX If locking the mutex fails, should we just stop execution? This
2025+ might be better than silently ignoring the error. */
2026+ __pthread_mutex_cond_lock (cbuffer->mutex);
2027+}
2028+
2029+/* This condvar implementation guarantees that all calls to signal and
2030+ broadcast and all of the three virtually atomic parts of each call to wait
2031+ (i.e., (1) releasing the mutex and blocking, (2) unblocking, and (3) re-
2032+ acquiring the mutex) happen in some total order that is consistent with the
2033+ happens-before relations in the calling program. However, this order does
2034+ not necessarily result in additional happens-before relations being
2035+ established (which aligns well with spurious wake-ups being allowed).
2036+
2037+ All waiters acquire a certain position in a 64b waiter sequence (__wseq).
2038+ This sequence determines which waiters are allowed to consume signals.
2039+ A broadcast is equal to sending as many signals as are unblocked waiters.
2040+ When a signal arrives, it samples the current value of __wseq with a
2041+ relaxed-MO load (i.e., the position the next waiter would get). (This is
2042+ sufficient because it is consistent with happens-before; the caller can
2043+ enforce stronger ordering constraints by calling signal while holding the
2044+ mutex.) Only waiters with a position less than the __wseq value observed
2045+ by the signal are eligible to consume this signal.
2046+
2047+ This would be straight-forward to implement if waiters would just spin but
2048+ we need to let them block using futexes. Futexes give no guarantee of
2049+ waking in FIFO order, so we cannot reliably wake eligible waiters if we
2050+ just use a single futex. Also, futex words are 32b in size, but we need
2051+ to distinguish more than 1<<32 states because we need to represent the
2052+ order of wake-up (and thus which waiters are eligible to consume signals);
2053+ blocking in a futex is not atomic with a waiter determining its position in
2054+ the waiter sequence, so we need the futex word to reliably notify waiters
2055+ that they should not attempt to block anymore because they have been
2056+ already signaled in the meantime. While an ABA issue on a 32b value will
2057+ be rare, ignoring it when we are aware of it is not the right thing to do
2058+ either.
2059+
2060+ Therefore, we use a 64b counter to represent the waiter sequence (on
2061+ architectures which only support 32b atomics, we use a few bits less).
2062+ To deal with the blocking using futexes, we maintain two groups of waiters:
2063+ * Group G1 consists of waiters that are all eligible to consume signals;
2064+ incoming signals will always signal waiters in this group until all
2065+ waiters in G1 have been signaled.
2066+ * Group G2 consists of waiters that arrive when a G1 is present and still
2067+ contains waiters that have not been signaled. When all waiters in G1
2068+ are signaled and a new signal arrives, the new signal will convert G2
2069+ into the new G1 and create a new G2 for future waiters.
2070+
2071+ We cannot allocate new memory because of process-shared condvars, so we
2072+ have just two slots of groups that change their role between G1 and G2.
2073+ Each has a separate futex word, a number of signals available for
2074+ consumption, a size (number of waiters in the group that have not been
2075+ signaled), and a reference count.
2076+
2077+ The group reference count is used to maintain the number of waiters that
2078+ are using the group's futex. Before a group can change its role, the
2079+ reference count must show that no waiters are using the futex anymore; this
2080+ prevents ABA issues on the futex word.
2081+
2082+ To represent which intervals in the waiter sequence the groups cover (and
2083+ thus also which group slot contains G1 or G2), we use a 64b counter to
2084+ designate the start position of G1 (inclusive), and a single bit in the
2085+ waiter sequence counter to represent which group slot currently contains
2086+ G2. This allows us to switch group roles atomically wrt. waiters obtaining
2087+ a position in the waiter sequence. The G1 start position allows waiters to
2088+ figure out whether they are in a group that has already been completely
2089+ signaled (i.e., if the current G1 starts at a later position that the
2090+ waiter's position). Waiters cannot determine whether they are currently
2091+ in G2 or G1 -- but they do not have too because all they are interested in
2092+ is whether there are available signals, and they always start in G2 (whose
2093+ group slot they know because of the bit in the waiter sequence. Signalers
2094+ will simply fill the right group until it is completely signaled and can
2095+ be closed (they do not switch group roles until they really have to to
2096+ decrease the likelihood of having to wait for waiters still holding a
2097+ reference on the now-closed G1).
2098+
2099+ Signalers maintain the initial size of G1 to be able to determine where
2100+ G2 starts (G2 is always open-ended until it becomes G1). They track the
2101+ remaining size of a group; when waiters cancel waiting (due to PThreads
2102+ cancellation or timeouts), they will decrease this remaining size as well.
2103+
2104+ To implement condvar destruction requirements (i.e., that
2105+ pthread_cond_destroy can be called as soon as all waiters have been
2106+ signaled), waiters increment a reference count before starting to wait and
2107+ decrement it after they stopped waiting but right before they acquire the
2108+ mutex associated with the condvar.
2109+
2110+ pthread_cond_t thus consists of the following (bits that are used for
2111+ flags and are not part of the primary value of each field but necessary
2112+ to make some things atomic or because there was no space for them
2113+ elsewhere in the data structure):
2114+
2115+ __wseq: Waiter sequence counter
2116+ * LSB is index of current G2.
2117+ * Waiters fetch-add while having acquire the mutex associated with the
2118+ condvar. Signalers load it and fetch-xor it concurrently.
2119+ __g1_start: Starting position of G1 (inclusive)
2120+ * LSB is index of current G2.
2121+ * Modified by signalers while having acquired the condvar-internal lock
2122+ and observed concurrently by waiters.
2123+ __g1_orig_size: Initial size of G1
2124+ * The two least-significant bits represent the condvar-internal lock.
2125+ * Only accessed while having acquired the condvar-internal lock.
2126+ __wrefs: Waiter reference counter.
2127+ * Bit 2 is true if waiters should run futex_wake when they remove the
2128+ last reference. pthread_cond_destroy uses this as futex word.
2129+ * Bit 1 is the clock ID (0 == CLOCK_REALTIME, 1 == CLOCK_MONOTONIC).
2130+ * Bit 0 is true iff this is a process-shared condvar.
2131+ * Simple reference count used by both waiters and pthread_cond_destroy.
2132+ (If the format of __wrefs is changed, update nptl_lock_constants.pysym
2133+ and the pretty printers.)
2134+ For each of the two groups, we have:
2135+ __g_refs: Futex waiter reference count.
2136+ * LSB is true if waiters should run futex_wake when they remove the
2137+ last reference.
2138+ * Reference count used by waiters concurrently with signalers that have
2139+ acquired the condvar-internal lock.
2140+ __g_signals: The number of signals that can still be consumed.
2141+ * Used as a futex word by waiters. Used concurrently by waiters and
2142+ signalers.
2143+ * LSB is true iff this group has been completely signaled (i.e., it is
2144+ closed).
2145+ __g_size: Waiters remaining in this group (i.e., which have not been
2146+ signaled yet.
2147+ * Accessed by signalers and waiters that cancel waiting (both do so only
2148+ when having acquired the condvar-internal lock.
2149+ * The size of G2 is always zero because it cannot be determined until
2150+ the group becomes G1.
2151+ * Although this is of unsigned type, we rely on using unsigned overflow
2152+ rules to make this hold effectively negative values too (in
2153+ particular, when waiters in G2 cancel waiting).
2154+
2155+ A PTHREAD_COND_INITIALIZER condvar has all fields set to zero, which yields
2156+ a condvar that has G2 starting at position 0 and a G1 that is closed.
2157+
2158+ Because waiters do not claim ownership of a group right when obtaining a
2159+ position in __wseq but only reference count the group when using futexes
2160+ to block, it can happen that a group gets closed before a waiter can
2161+ increment the reference count. Therefore, waiters have to check whether
2162+ their group is already closed using __g1_start. They also have to perform
2163+ this check when spinning when trying to grab a signal from __g_signals.
2164+ Note that for these checks, using relaxed MO to load __g1_start is
2165+ sufficient because if a waiter can see a sufficiently large value, it could
2166+ have also consume a signal in the waiters group.
2167+
2168+ Waiters try to grab a signal from __g_signals without holding a reference
2169+ count, which can lead to stealing a signal from a more recent group after
2170+ their own group was already closed. They cannot always detect whether they
2171+ in fact did because they do not know when they stole, but they can
2172+ conservatively add a signal back to the group they stole from; if they
2173+ did so unnecessarily, all that happens is a spurious wake-up. To make this
2174+ even less likely, __g1_start contains the index of the current g2 too,
2175+ which allows waiters to check if there aliasing on the group slots; if
2176+ there wasn't, they didn't steal from the current G1, which means that the
2177+ G1 they stole from must have been already closed and they do not need to
2178+ fix anything.
2179+
2180+ It is essential that the last field in pthread_cond_t is __g_signals[1]:
2181+ The previous condvar used a pointer-sized field in pthread_cond_t, so a
2182+ PTHREAD_COND_INITIALIZER from that condvar implementation might only
2183+ initialize 4 bytes to zero instead of the 8 bytes we need (i.e., 44 bytes
2184+ in total instead of the 48 we need). __g_signals[1] is not accessed before
2185+ the first group switch (G2 starts at index 0), which will set its value to
2186+ zero after a harmless fetch-or whose return value is ignored. This
2187+ effectively completes initialization.
2188+
2189+
2190+ Limitations:
2191+ * This condvar isn't designed to allow for more than
2192+ __PTHREAD_COND_MAX_GROUP_SIZE * (1 << 31) calls to __pthread_cond_wait.
2193+ * More than __PTHREAD_COND_MAX_GROUP_SIZE concurrent waiters are not
2194+ supported.
2195+ * Beyond what is allowed as errors by POSIX or documented, we can also
2196+ return the following errors:
2197+ * EPERM if MUTEX is a recursive mutex and the caller doesn't own it.
2198+ * EOWNERDEAD or ENOTRECOVERABLE when using robust mutexes. Unlike
2199+ for other errors, this can happen when we re-acquire the mutex; this
2200+ isn't allowed by POSIX (which requires all errors to virtually happen
2201+ before we release the mutex or change the condvar state), but there's
2202+ nothing we can do really.
2203+ * When using PTHREAD_MUTEX_PP_* mutexes, we can also return all errors
2204+ returned by __pthread_tpp_change_priority. We will already have
2205+ released the mutex in such cases, so the caller cannot expect to own
2206+ MUTEX.
2207+
2208+ Other notes:
2209+ * Instead of the normal mutex unlock / lock functions, we use
2210+ __pthread_mutex_unlock_usercnt(m, 0) / __pthread_mutex_cond_lock(m)
2211+ because those will not change the mutex-internal users count, so that it
2212+ can be detected when a condvar is still associated with a particular
2213+ mutex because there is a waiter blocked on this condvar using this mutex.
2214+*/
2215+static __always_inline int
2216+__pthread_cond_wait_common (pthread_cond_t *cond, pthread_mutex_t *mutex,
2217+ const struct timespec *abstime)
2218 {
2219- struct _pthread_cleanup_buffer buffer;
2220- struct _condvar_cleanup_buffer cbuffer;
2221+ const int maxspin = 0;
2222 int err;
2223- int pshared = (cond->__data.__mutex == (void *) ~0l)
2224- ? LLL_SHARED : LLL_PRIVATE;
2225-
2226-#if (defined lll_futex_wait_requeue_pi \
2227- && defined __ASSUME_REQUEUE_PI)
2228- int pi_flag = 0;
2229-#endif
2230+ int result = 0;
2231
2232 LIBC_PROBE (cond_wait, 2, cond, mutex);
2233
2234- /* Make sure we are alone. */
2235- lll_lock (cond->__data.__lock, pshared);
2236-
2237- /* Now we can release the mutex. */
2238+ /* Acquire a position (SEQ) in the waiter sequence (WSEQ). We use an
2239+ atomic operation because signals and broadcasts may update the group
2240+ switch without acquiring the mutex. We do not need release MO here
2241+ because we do not need to establish any happens-before relation with
2242+ signalers (see __pthread_cond_signal); modification order alone
2243+ establishes a total order of waiters/signals. We do need acquire MO
2244+ to synchronize with group reinitialization in
2245+ __condvar_quiesce_and_switch_g1. */
2246+ uint64_t wseq = __condvar_fetch_add_wseq_acquire (cond, 2);
2247+ /* Find our group's index. We always go into what was G2 when we acquired
2248+ our position. */
2249+ unsigned int g = wseq & 1;
2250+ uint64_t seq = wseq >> 1;
2251+
2252+ /* Increase the waiter reference count. Relaxed MO is sufficient because
2253+ we only need to synchronize when decrementing the reference count. */
2254+ unsigned int flags = atomic_fetch_add_relaxed (&cond->__data.__wrefs, 8);
2255+ int private = __condvar_get_private (flags);
2256+
2257+ /* Now that we are registered as a waiter, we can release the mutex.
2258+ Waiting on the condvar must be atomic with releasing the mutex, so if
2259+ the mutex is used to establish a happens-before relation with any
2260+ signaler, the waiter must be visible to the latter; thus, we release the
2261+ mutex after registering as waiter.
2262+ If releasing the mutex fails, we just cancel our registration as a
2263+ waiter and confirm that we have woken up. */
2264 err = __pthread_mutex_unlock_usercnt (mutex, 0);
2265- if (__glibc_unlikely (err))
2266+ if (__glibc_unlikely (err != 0))
2267 {
2268- lll_unlock (cond->__data.__lock, pshared);
2269+ __condvar_cancel_waiting (cond, seq, g, private);
2270+ __condvar_confirm_wakeup (cond, private);
2271 return err;
2272 }
2273
2274- /* We have one new user of the condvar. */
2275- ++cond->__data.__total_seq;
2276- ++cond->__data.__futex;
2277- cond->__data.__nwaiters += 1 << COND_NWAITERS_SHIFT;
2278-
2279- /* Remember the mutex we are using here. If there is already a
2280- different address store this is a bad user bug. Do not store
2281- anything for pshared condvars. */
2282- if (cond->__data.__mutex != (void *) ~0l)
2283- cond->__data.__mutex = mutex;
2284-
2285- /* Prepare structure passed to cancellation handler. */
2286- cbuffer.cond = cond;
2287- cbuffer.mutex = mutex;
2288-
2289- /* Before we block we enable cancellation. Therefore we have to
2290- install a cancellation handler. */
2291- __pthread_cleanup_push (&buffer, __condvar_cleanup, &cbuffer);
2292-
2293- /* The current values of the wakeup counter. The "woken" counter
2294- must exceed this value. */
2295- unsigned long long int val;
2296- unsigned long long int seq;
2297- val = seq = cond->__data.__wakeup_seq;
2298- /* Remember the broadcast counter. */
2299- cbuffer.bc_seq = cond->__data.__broadcast_seq;
2300+ /* Now wait until a signal is available in our group or it is closed.
2301+ Acquire MO so that if we observe a value of zero written after group
2302+ switching in __condvar_quiesce_and_switch_g1, we synchronize with that
2303+ store and will see the prior update of __g1_start done while switching
2304+ groups too. */
2305+ unsigned int signals = atomic_load_acquire (cond->__data.__g_signals + g);
2306
2307 do
2308 {
2309- unsigned int futex_val = cond->__data.__futex;
2310- /* Prepare to wait. Release the condvar futex. */
2311- lll_unlock (cond->__data.__lock, pshared);
2312-
2313- /* Enable asynchronous cancellation. Required by the standard. */
2314- cbuffer.oldtype = __pthread_enable_asynccancel ();
2315-
2316-#if (defined lll_futex_wait_requeue_pi \
2317- && defined __ASSUME_REQUEUE_PI)
2318- /* If pi_flag remained 1 then it means that we had the lock and the mutex
2319- but a spurious waker raced ahead of us. Give back the mutex before
2320- going into wait again. */
2321- if (pi_flag)
2322+ while (1)
2323 {
2324- __pthread_mutex_cond_lock_adjust (mutex);
2325- __pthread_mutex_unlock_usercnt (mutex, 0);
2326+ /* Spin-wait first.
2327+ Note that spinning first without checking whether a timeout
2328+ passed might lead to what looks like a spurious wake-up even
2329+ though we should return ETIMEDOUT (e.g., if the caller provides
2330+ an absolute timeout that is clearly in the past). However,
2331+ (1) spurious wake-ups are allowed, (2) it seems unlikely that a
2332+ user will (ab)use pthread_cond_wait as a check for whether a
2333+ point in time is in the past, and (3) spinning first without
2334+ having to compare against the current time seems to be the right
2335+ choice from a performance perspective for most use cases. */
2336+ unsigned int spin = maxspin;
2337+ while (signals == 0 && spin > 0)
2338+ {
2339+ /* Check that we are not spinning on a group that's already
2340+ closed. */
2341+ if (seq < (__condvar_load_g1_start_relaxed (cond) >> 1))
2342+ goto done;
2343+
2344+ /* TODO Back off. */
2345+
2346+ /* Reload signals. See above for MO. */
2347+ signals = atomic_load_acquire (cond->__data.__g_signals + g);
2348+ spin--;
2349+ }
2350+
2351+ /* If our group will be closed as indicated by the flag on signals,
2352+ don't bother grabbing a signal. */
2353+ if (signals & 1)
2354+ goto done;
2355+
2356+ /* If there is an available signal, don't block. */
2357+ if (signals != 0)
2358+ break;
2359+
2360+ /* No signals available after spinning, so prepare to block.
2361+ We first acquire a group reference and use acquire MO for that so
2362+ that we synchronize with the dummy read-modify-write in
2363+ __condvar_quiesce_and_switch_g1 if we read from that. In turn,
2364+ in this case this will make us see the closed flag on __g_signals
2365+ that designates a concurrent attempt to reuse the group's slot.
2366+ We use acquire MO for the __g_signals check to make the
2367+ __g1_start check work (see spinning above).
2368+ Note that the group reference acquisition will not mask the
2369+ release MO when decrementing the reference count because we use
2370+ an atomic read-modify-write operation and thus extend the release
2371+ sequence. */
2372+ atomic_fetch_add_acquire (cond->__data.__g_refs + g, 2);
2373+ if (((atomic_load_acquire (cond->__data.__g_signals + g) & 1) != 0)
2374+ || (seq < (__condvar_load_g1_start_relaxed (cond) >> 1)))
2375+ {
2376+ /* Our group is closed. Wake up any signalers that might be
2377+ waiting. */
2378+ __condvar_dec_grefs (cond, g, private);
2379+ goto done;
2380+ }
2381+
2382+ // Now block.
2383+ struct _pthread_cleanup_buffer buffer;
2384+ struct _condvar_cleanup_buffer cbuffer;
2385+ cbuffer.wseq = wseq;
2386+ cbuffer.cond = cond;
2387+ cbuffer.mutex = mutex;
2388+ cbuffer.private = private;
2389+ __pthread_cleanup_push (&buffer, __condvar_cleanup_waiting, &cbuffer);
2390+
2391+ if (abstime == NULL)
2392+ {
2393+ /* Block without a timeout. */
2394+ err = futex_wait_cancelable (
2395+ cond->__data.__g_signals + g, 0, private);
2396+ }
2397+ else
2398+ {
2399+ /* Block, but with a timeout.
2400+ Work around the fact that the kernel rejects negative timeout
2401+ values despite them being valid. */
2402+ if (__glibc_unlikely (abstime->tv_sec < 0))
2403+ err = ETIMEDOUT;
2404+
2405+ else if ((flags & __PTHREAD_COND_CLOCK_MONOTONIC_MASK) != 0)
2406+ {
2407+ /* CLOCK_MONOTONIC is requested. */
2408+ struct timespec rt;
2409+ if (__clock_gettime (CLOCK_MONOTONIC, &rt) != 0)
2410+ __libc_fatal ("clock_gettime does not support "
2411+ "CLOCK_MONOTONIC");
2412+ /* Convert the absolute timeout value to a relative
2413+ timeout. */
2414+ rt.tv_sec = abstime->tv_sec - rt.tv_sec;
2415+ rt.tv_nsec = abstime->tv_nsec - rt.tv_nsec;
2416+ if (rt.tv_nsec < 0)
2417+ {
2418+ rt.tv_nsec += 1000000000;
2419+ --rt.tv_sec;
2420+ }
2421+ /* Did we already time out? */
2422+ if (__glibc_unlikely (rt.tv_sec < 0))
2423+ err = ETIMEDOUT;
2424+ else
2425+ err = futex_reltimed_wait_cancelable
2426+ (cond->__data.__g_signals + g, 0, &rt, private);
2427+ }
2428+ else
2429+ {
2430+ /* Use CLOCK_REALTIME. */
2431+ err = futex_abstimed_wait_cancelable
2432+ (cond->__data.__g_signals + g, 0, abstime, private);
2433+ }
2434+ }
2435+
2436+ __pthread_cleanup_pop (&buffer, 0);
2437+
2438+ if (__glibc_unlikely (err == ETIMEDOUT))
2439+ {
2440+ __condvar_dec_grefs (cond, g, private);
2441+ /* If we timed out, we effectively cancel waiting. Note that
2442+ we have decremented __g_refs before cancellation, so that a
2443+ deadlock between waiting for quiescence of our group in
2444+ __condvar_quiesce_and_switch_g1 and us trying to acquire
2445+ the lock during cancellation is not possible. */
2446+ __condvar_cancel_waiting (cond, seq, g, private);
2447+ result = ETIMEDOUT;
2448+ goto done;
2449+ }
2450+ else
2451+ __condvar_dec_grefs (cond, g, private);
2452+
2453+ /* Reload signals. See above for MO. */
2454+ signals = atomic_load_acquire (cond->__data.__g_signals + g);
2455 }
2456- pi_flag = USE_REQUEUE_PI (mutex);
2457
2458- if (pi_flag)
2459+ }
2460+ /* Try to grab a signal. Use acquire MO so that we see an up-to-date value
2461+ of __g1_start below (see spinning above for a similar case). In
2462+ particular, if we steal from a more recent group, we will also see a
2463+ more recent __g1_start below. */
2464+ while (!atomic_compare_exchange_weak_acquire (cond->__data.__g_signals + g,
2465+ &signals, signals - 2));
2466+
2467+ /* We consumed a signal but we could have consumed from a more recent group
2468+ that aliased with ours due to being in the same group slot. If this
2469+ might be the case our group must be closed as visible through
2470+ __g1_start. */
2471+ uint64_t g1_start = __condvar_load_g1_start_relaxed (cond);
2472+ if (seq < (g1_start >> 1))
2473+ {
2474+ /* We potentially stole a signal from a more recent group but we do not
2475+ know which group we really consumed from.
2476+ We do not care about groups older than current G1 because they are
2477+ closed; we could have stolen from these, but then we just add a
2478+ spurious wake-up for the current groups.
2479+ We will never steal a signal from current G2 that was really intended
2480+ for G2 because G2 never receives signals (until it becomes G1). We
2481+ could have stolen a signal from G2 that was conservatively added by a
2482+ previous waiter that also thought it stole a signal -- but given that
2483+ that signal was added unnecessarily, it's not a problem if we steal
2484+ it.
2485+ Thus, the remaining case is that we could have stolen from the current
2486+ G1, where "current" means the __g1_start value we observed. However,
2487+ if the current G1 does not have the same slot index as we do, we did
2488+ not steal from it and do not need to undo that. This is the reason
2489+ for putting a bit with G2's index into__g1_start as well. */
2490+ if (((g1_start & 1) ^ 1) == g)
2491 {
2492- err = lll_futex_wait_requeue_pi (&cond->__data.__futex,
2493- futex_val, &mutex->__data.__lock,
2494- pshared);
2495-
2496- pi_flag = (err == 0);
2497+ /* We have to conservatively undo our potential mistake of stealing
2498+ a signal. We can stop trying to do that when the current G1
2499+ changes because other spinning waiters will notice this too and
2500+ __condvar_quiesce_and_switch_g1 has checked that there are no
2501+ futex waiters anymore before switching G1.
2502+ Relaxed MO is fine for the __g1_start load because we need to
2503+ merely be able to observe this fact and not have to observe
2504+ something else as well.
2505+ ??? Would it help to spin for a little while to see whether the
2506+ current G1 gets closed? This might be worthwhile if the group is
2507+ small or close to being closed. */
2508+ unsigned int s = atomic_load_relaxed (cond->__data.__g_signals + g);
2509+ while (__condvar_load_g1_start_relaxed (cond) == g1_start)
2510+ {
2511+ /* Try to add a signal. We don't need to acquire the lock
2512+ because at worst we can cause a spurious wake-up. If the
2513+ group is in the process of being closed (LSB is true), this
2514+ has an effect similar to us adding a signal. */
2515+ if (((s & 1) != 0)
2516+ || atomic_compare_exchange_weak_relaxed
2517+ (cond->__data.__g_signals + g, &s, s + 2))
2518+ {
2519+ /* If we added a signal, we also need to add a wake-up on
2520+ the futex. We also need to do that if we skipped adding
2521+ a signal because the group is being closed because
2522+ while __condvar_quiesce_and_switch_g1 could have closed
2523+ the group, it might stil be waiting for futex waiters to
2524+ leave (and one of those waiters might be the one we stole
2525+ the signal from, which cause it to block using the
2526+ futex). */
2527+ futex_wake (cond->__data.__g_signals + g, 1, private);
2528+ break;
2529+ }
2530+ /* TODO Back off. */
2531+ }
2532 }
2533- else
2534-#endif
2535- /* Wait until woken by signal or broadcast. */
2536- lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
2537-
2538- /* Disable asynchronous cancellation. */
2539- __pthread_disable_asynccancel (cbuffer.oldtype);
2540-
2541- /* We are going to look at shared data again, so get the lock. */
2542- lll_lock (cond->__data.__lock, pshared);
2543-
2544- /* If a broadcast happened, we are done. */
2545- if (cbuffer.bc_seq != cond->__data.__broadcast_seq)
2546- goto bc_out;
2547-
2548- /* Check whether we are eligible for wakeup. */
2549- val = cond->__data.__wakeup_seq;
2550 }
2551- while (val == seq || cond->__data.__woken_seq == val);
2552
2553- /* Another thread woken up. */
2554- ++cond->__data.__woken_seq;
2555+ done:
2556
2557- bc_out:
2558+ /* Confirm that we have been woken. We do that before acquiring the mutex
2559+ to allow for execution of pthread_cond_destroy while having acquired the
2560+ mutex. */
2561+ __condvar_confirm_wakeup (cond, private);
2562
2563- cond->__data.__nwaiters -= 1 << COND_NWAITERS_SHIFT;
2564-
2565- /* If pthread_cond_destroy was called on this varaible already,
2566- notify the pthread_cond_destroy caller all waiters have left
2567- and it can be successfully destroyed. */
2568- if (cond->__data.__total_seq == -1ULL
2569- && cond->__data.__nwaiters < (1 << COND_NWAITERS_SHIFT))
2570- lll_futex_wake (&cond->__data.__nwaiters, 1, pshared);
2571+ /* Woken up; now re-acquire the mutex. If this doesn't fail, return RESULT,
2572+ which is set to ETIMEDOUT if a timeout occured, or zero otherwise. */
2573+ err = __pthread_mutex_cond_lock (mutex);
2574+ /* XXX Abort on errors that are disallowed by POSIX? */
2575+ return (err != 0) ? err : result;
2576+}
2577
2578- /* We are done with the condvar. */
2579- lll_unlock (cond->__data.__lock, pshared);
2580
2581- /* The cancellation handling is back to normal, remove the handler. */
2582- __pthread_cleanup_pop (&buffer, 0);
2583+/* See __pthread_cond_wait_common. */
2584+int
2585+__pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
2586+{
2587+ return __pthread_cond_wait_common (cond, mutex, NULL);
2588+}
2589
2590- /* Get the mutex before returning. Not needed for PI. */
2591-#if (defined lll_futex_wait_requeue_pi \
2592- && defined __ASSUME_REQUEUE_PI)
2593- if (pi_flag)
2594- {
2595- __pthread_mutex_cond_lock_adjust (mutex);
2596- return 0;
2597- }
2598- else
2599-#endif
2600- return __pthread_mutex_cond_lock (mutex);
2601+/* See __pthread_cond_wait_common. */
2602+int
2603+__pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
2604+ const struct timespec *abstime)
2605+{
2606+ /* Check parameter validity. This should also tell the compiler that
2607+ it can assume that abstime is not NULL. */
2608+ if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)
2609+ return EINVAL;
2610+ return __pthread_cond_wait_common (cond, mutex, abstime);
2611 }
2612
2613 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
2614 GLIBC_2_3_2);
2615+versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
2616+ GLIBC_2_3_2);
2617diff --git a/nptl/pthread_condattr_getclock.c b/nptl/pthread_condattr_getclock.c
2618index d156302..cecb4aa 100644
2619--- a/nptl/pthread_condattr_getclock.c
2620+++ b/nptl/pthread_condattr_getclock.c
2621@@ -23,6 +23,6 @@ int
2622 pthread_condattr_getclock (const pthread_condattr_t *attr, clockid_t *clock_id)
2623 {
2624 *clock_id = (((((const struct pthread_condattr *) attr)->value) >> 1)
2625- & ((1 << COND_NWAITERS_SHIFT) - 1));
2626+ & ((1 << COND_CLOCK_BITS) - 1));
2627 return 0;
2628 }
2629diff --git a/nptl/pthread_condattr_getpshared.c b/nptl/pthread_condattr_getpshared.c
2630index 5a10f3e..8147966 100644
2631--- a/nptl/pthread_condattr_getpshared.c
2632+++ b/nptl/pthread_condattr_getpshared.c
2633@@ -22,7 +22,8 @@
2634 int
2635 pthread_condattr_getpshared (const pthread_condattr_t *attr, int *pshared)
2636 {
2637- *pshared = ((const struct pthread_condattr *) attr)->value & 1;
2638+ *pshared = (((const struct pthread_condattr *) attr)->value & 1
2639+ ? PTHREAD_PROCESS_SHARED : PTHREAD_PROCESS_PRIVATE);
2640
2641 return 0;
2642 }
2643diff --git a/nptl/pthread_condattr_init.c b/nptl/pthread_condattr_init.c
2644index 0ce42e5..6e5168d 100644
2645--- a/nptl/pthread_condattr_init.c
2646+++ b/nptl/pthread_condattr_init.c
2647@@ -23,7 +23,9 @@
2648 int
2649 __pthread_condattr_init (pthread_condattr_t *attr)
2650 {
2651- memset (attr, '\0', sizeof (*attr));
2652+ struct pthread_condattr *iattr = (struct pthread_condattr *) attr;
2653+ /* Default is not pshared and CLOCK_REALTIME. */
2654+ iattr-> value = CLOCK_REALTIME << 1;
2655
2656 return 0;
2657 }
2658diff --git a/nptl/pthread_condattr_setclock.c b/nptl/pthread_condattr_setclock.c
2659index 25e2a17..3cfad84 100644
2660--- a/nptl/pthread_condattr_setclock.c
2661+++ b/nptl/pthread_condattr_setclock.c
2662@@ -18,7 +18,7 @@
2663
2664 #include <assert.h>
2665 #include <errno.h>
2666-#include <stdbool.h>
2667+#include <futex-internal.h>
2668 #include <time.h>
2669 #include <sysdep.h>
2670 #include "pthreadP.h"
2671@@ -33,12 +33,17 @@ pthread_condattr_setclock (pthread_condattr_t *attr, clockid_t clock_id)
2672 in the pthread_cond_t structure needs to be adjusted. */
2673 return EINVAL;
2674
2675+ /* If we do not support waiting using CLOCK_MONOTONIC, return an error. */
2676+ if (clock_id == CLOCK_MONOTONIC
2677+ && !futex_supports_exact_relative_timeouts())
2678+ return ENOTSUP;
2679+
2680 /* Make sure the value fits in the bits we reserved. */
2681- assert (clock_id < (1 << COND_NWAITERS_SHIFT));
2682+ assert (clock_id < (1 << COND_CLOCK_BITS));
2683
2684 int *valuep = &((struct pthread_condattr *) attr)->value;
2685
2686- *valuep = ((*valuep & ~(((1 << COND_NWAITERS_SHIFT) - 1) << 1))
2687+ *valuep = ((*valuep & ~(((1 << COND_CLOCK_BITS) - 1) << 1))
2688 | (clock_id << 1));
2689
2690 return 0;
2691diff --git a/nptl/test-cond-printers.py b/nptl/test-cond-printers.py
2692index af0e12e..9e807c9 100644
2693--- a/nptl/test-cond-printers.py
2694+++ b/nptl/test-cond-printers.py
2695@@ -35,7 +35,7 @@ try:
2696
2697 break_at(test_source, 'Test status (destroyed)')
2698 continue_cmd() # Go to test_status_destroyed
2699- test_printer(var, to_string, {'Status': 'Destroyed'})
2700+ test_printer(var, to_string, {'Threads known to still execute a wait function': '0'})
2701
2702 continue_cmd() # Exit
2703
2704diff --git a/nptl/tst-cond1.c b/nptl/tst-cond1.c
2705index 75ab9c8..509bbd0 100644
2706--- a/nptl/tst-cond1.c
2707+++ b/nptl/tst-cond1.c
2708@@ -73,6 +73,9 @@ do_test (void)
2709
2710 puts ("parent: wait for condition");
2711
2712+ /* This test will fail on spurious wake-ups, which are allowed; however,
2713+ the current implementation shouldn't produce spurious wake-ups in the
2714+ scenario we are testing here. */
2715 err = pthread_cond_wait (&cond, &mut);
2716 if (err != 0)
2717 error (EXIT_FAILURE, err, "parent: cannot wait fir signal");
2718diff --git a/nptl/tst-cond20.c b/nptl/tst-cond20.c
2719index 918c4ad..665a66a 100644
2720--- a/nptl/tst-cond20.c
2721+++ b/nptl/tst-cond20.c
2722@@ -96,7 +96,10 @@ do_test (void)
2723
2724 for (i = 0; i < ROUNDS; ++i)
2725 {
2726- pthread_cond_wait (&cond2, &mut);
2727+ /* Make sure we discard spurious wake-ups. */
2728+ do
2729+ pthread_cond_wait (&cond2, &mut);
2730+ while (count != N);
2731
2732 if (i & 1)
2733 pthread_mutex_unlock (&mut);
2734diff --git a/nptl/tst-cond22.c b/nptl/tst-cond22.c
2735index bd978e5..64f19ea 100644
2736--- a/nptl/tst-cond22.c
2737+++ b/nptl/tst-cond22.c
2738@@ -106,10 +106,11 @@ do_test (void)
2739 status = 1;
2740 }
2741
2742- printf ("cond = { %d, %x, %lld, %lld, %lld, %p, %u, %u }\n",
2743- c.__data.__lock, c.__data.__futex, c.__data.__total_seq,
2744- c.__data.__wakeup_seq, c.__data.__woken_seq, c.__data.__mutex,
2745- c.__data.__nwaiters, c.__data.__broadcast_seq);
2746+ printf ("cond = { %llu, %llu, %u/%u/%u, %u/%u/%u, %u, %u }\n",
2747+ c.__data.__wseq, c.__data.__g1_start,
2748+ c.__data.__g_signals[0], c.__data.__g_refs[0], c.__data.__g_size[0],
2749+ c.__data.__g_signals[1], c.__data.__g_refs[1], c.__data.__g_size[1],
2750+ c.__data.__g1_orig_size, c.__data.__wrefs);
2751
2752 if (pthread_create (&th, NULL, tf, (void *) 1l) != 0)
2753 {
2754@@ -148,10 +149,11 @@ do_test (void)
2755 status = 1;
2756 }
2757
2758- printf ("cond = { %d, %x, %lld, %lld, %lld, %p, %u, %u }\n",
2759- c.__data.__lock, c.__data.__futex, c.__data.__total_seq,
2760- c.__data.__wakeup_seq, c.__data.__woken_seq, c.__data.__mutex,
2761- c.__data.__nwaiters, c.__data.__broadcast_seq);
2762+ printf ("cond = { %llu, %llu, %u/%u/%u, %u/%u/%u, %u, %u }\n",
2763+ c.__data.__wseq, c.__data.__g1_start,
2764+ c.__data.__g_signals[0], c.__data.__g_refs[0], c.__data.__g_size[0],
2765+ c.__data.__g_signals[1], c.__data.__g_refs[1], c.__data.__g_size[1],
2766+ c.__data.__g1_orig_size, c.__data.__wrefs);
2767
2768 return status;
2769 }
2770diff --git a/sysdeps/aarch64/nptl/bits/pthreadtypes.h b/sysdeps/aarch64/nptl/bits/pthreadtypes.h
2771index 13984a7..c6fa632 100644
2772--- a/sysdeps/aarch64/nptl/bits/pthreadtypes.h
2773+++ b/sysdeps/aarch64/nptl/bits/pthreadtypes.h
2774@@ -90,17 +90,30 @@ typedef union
2775 {
2776 struct
2777 {
2778- int __lock;
2779- unsigned int __futex;
2780- __extension__ unsigned long long int __total_seq;
2781- __extension__ unsigned long long int __wakeup_seq;
2782- __extension__ unsigned long long int __woken_seq;
2783- void *__mutex;
2784- unsigned int __nwaiters;
2785- unsigned int __broadcast_seq;
2786+ __extension__ union
2787+ {
2788+ __extension__ unsigned long long int __wseq;
2789+ struct {
2790+ unsigned int __low;
2791+ unsigned int __high;
2792+ } __wseq32;
2793+ };
2794+ __extension__ union
2795+ {
2796+ __extension__ unsigned long long int __g1_start;
2797+ struct {
2798+ unsigned int __low;
2799+ unsigned int __high;
2800+ } __g1_start32;
2801+ };
2802+ unsigned int __g_refs[2];
2803+ unsigned int __g_size[2];
2804+ unsigned int __g1_orig_size;
2805+ unsigned int __wrefs;
2806+ unsigned int __g_signals[2];
2807 } __data;
2808 char __size[__SIZEOF_PTHREAD_COND_T];
2809- long int __align;
2810+ __extension__ long long int __align;
2811 } pthread_cond_t;
2812
2813 typedef union
2814diff --git a/sysdeps/arm/nptl/bits/pthreadtypes.h b/sysdeps/arm/nptl/bits/pthreadtypes.h
2815index afb5392..53518c6 100644
2816--- a/sysdeps/arm/nptl/bits/pthreadtypes.h
2817+++ b/sysdeps/arm/nptl/bits/pthreadtypes.h
2818@@ -93,14 +93,27 @@ typedef union
2819 {
2820 struct
2821 {
2822- int __lock;
2823- unsigned int __futex;
2824- __extension__ unsigned long long int __total_seq;
2825- __extension__ unsigned long long int __wakeup_seq;
2826- __extension__ unsigned long long int __woken_seq;
2827- void *__mutex;
2828- unsigned int __nwaiters;
2829- unsigned int __broadcast_seq;
2830+ __extension__ union
2831+ {
2832+ __extension__ unsigned long long int __wseq;
2833+ struct {
2834+ unsigned int __low;
2835+ unsigned int __high;
2836+ } __wseq32;
2837+ };
2838+ __extension__ union
2839+ {
2840+ __extension__ unsigned long long int __g1_start;
2841+ struct {
2842+ unsigned int __low;
2843+ unsigned int __high;
2844+ } __g1_start32;
2845+ };
2846+ unsigned int __g_refs[2];
2847+ unsigned int __g_size[2];
2848+ unsigned int __g1_orig_size;
2849+ unsigned int __wrefs;
2850+ unsigned int __g_signals[2];
2851 } __data;
2852 char __size[__SIZEOF_PTHREAD_COND_T];
2853 __extension__ long long int __align;
2854diff --git a/sysdeps/ia64/nptl/bits/pthreadtypes.h b/sysdeps/ia64/nptl/bits/pthreadtypes.h
2855index f2e6dac..e72dbfd 100644
2856--- a/sysdeps/ia64/nptl/bits/pthreadtypes.h
2857+++ b/sysdeps/ia64/nptl/bits/pthreadtypes.h
2858@@ -90,17 +90,30 @@ typedef union
2859 {
2860 struct
2861 {
2862- int __lock;
2863- unsigned int __futex;
2864- __extension__ unsigned long long int __total_seq;
2865- __extension__ unsigned long long int __wakeup_seq;
2866- __extension__ unsigned long long int __woken_seq;
2867- void *__mutex;
2868- unsigned int __nwaiters;
2869- unsigned int __broadcast_seq;
2870+ __extension__ union
2871+ {
2872+ __extension__ unsigned long long int __wseq;
2873+ struct {
2874+ unsigned int __low;
2875+ unsigned int __high;
2876+ } __wseq32;
2877+ };
2878+ __extension__ union
2879+ {
2880+ __extension__ unsigned long long int __g1_start;
2881+ struct {
2882+ unsigned int __low;
2883+ unsigned int __high;
2884+ } __g1_start32;
2885+ };
2886+ unsigned int __g_refs[2];
2887+ unsigned int __g_size[2];
2888+ unsigned int __g1_orig_size;
2889+ unsigned int __wrefs;
2890+ unsigned int __g_signals[2];
2891 } __data;
2892 char __size[__SIZEOF_PTHREAD_COND_T];
2893- long int __align;
2894+ __extension__ long long int __align;
2895 } pthread_cond_t;
2896
2897 typedef union
2898diff --git a/sysdeps/m68k/nptl/bits/pthreadtypes.h b/sysdeps/m68k/nptl/bits/pthreadtypes.h
2899index d8faa7a..c5e9021 100644
2900--- a/sysdeps/m68k/nptl/bits/pthreadtypes.h
2901+++ b/sysdeps/m68k/nptl/bits/pthreadtypes.h
2902@@ -88,19 +88,33 @@ typedef union
2903
2904
2905 /* Data structure for conditional variable handling. The structure of
2906- the attribute type is deliberately not exposed. */
2907+ the attribute type is not exposed on purpose. */
2908 typedef union
2909 {
2910 struct
2911 {
2912- int __lock __attribute__ ((__aligned__ (4)));
2913- unsigned int __futex;
2914- __extension__ unsigned long long int __total_seq;
2915- __extension__ unsigned long long int __wakeup_seq;
2916- __extension__ unsigned long long int __woken_seq;
2917- void *__mutex;
2918- unsigned int __nwaiters;
2919- unsigned int __broadcast_seq;
2920+ __extension__ union
2921+ {
2922+ __extension__ unsigned long long int __wseq;
2923+ struct {
2924+ unsigned int __low;
2925+ unsigned int __high;
2926+ } __wseq32;
2927+ };
2928+ __extension__ union
2929+ {
2930+ __extension__ unsigned long long int __g1_start;
2931+ struct {
2932+ unsigned int __low;
2933+ unsigned int __high;
2934+ } __g1_start32;
2935+ };
2936+ /* Enforce proper alignment of fields used as futex words. */
2937+ unsigned int __g_refs[2] __attribute__ ((__aligned__ (4)));
2938+ unsigned int __g_size[2];
2939+ unsigned int __g1_orig_size;
2940+ unsigned int __wrefs;
2941+ unsigned int __g_signals[2];
2942 } __data;
2943 char __size[__SIZEOF_PTHREAD_COND_T];
2944 __extension__ long long int __align;
2945diff --git a/sysdeps/microblaze/nptl/bits/pthreadtypes.h b/sysdeps/microblaze/nptl/bits/pthreadtypes.h
2946index 9e9e307..b6623c2 100644
2947--- a/sysdeps/microblaze/nptl/bits/pthreadtypes.h
2948+++ b/sysdeps/microblaze/nptl/bits/pthreadtypes.h
2949@@ -91,14 +91,27 @@ typedef union
2950 {
2951 struct
2952 {
2953- int __lock;
2954- unsigned int __futex;
2955- __extension__ unsigned long long int __total_seq;
2956- __extension__ unsigned long long int __wakeup_seq;
2957- __extension__ unsigned long long int __woken_seq;
2958- void *__mutex;
2959- unsigned int __nwaiters;
2960- unsigned int __broadcast_seq;
2961+ __extension__ union
2962+ {
2963+ __extension__ unsigned long long int __wseq;
2964+ struct {
2965+ unsigned int __low;
2966+ unsigned int __high;
2967+ } __wseq32;
2968+ };
2969+ __extension__ union
2970+ {
2971+ __extension__ unsigned long long int __g1_start;
2972+ struct {
2973+ unsigned int __low;
2974+ unsigned int __high;
2975+ } __g1_start32;
2976+ };
2977+ unsigned int __g_refs[2];
2978+ unsigned int __g_size[2];
2979+ unsigned int __g1_orig_size;
2980+ unsigned int __wrefs;
2981+ unsigned int __g_signals[2];
2982 } __data;
2983 char __size[__SIZEOF_PTHREAD_COND_T];
2984 __extension__ long long int __align;
2985diff --git a/sysdeps/mips/nptl/bits/pthreadtypes.h b/sysdeps/mips/nptl/bits/pthreadtypes.h
2986index 68ed94b..7ddc7bf 100644
2987--- a/sysdeps/mips/nptl/bits/pthreadtypes.h
2988+++ b/sysdeps/mips/nptl/bits/pthreadtypes.h
2989@@ -117,19 +117,32 @@ typedef union
2990
2991
2992 /* Data structure for conditional variable handling. The structure of
2993- the attribute type is deliberately not exposed. */
2994+ the attribute type is not exposed on purpose. */
2995 typedef union
2996 {
2997 struct
2998 {
2999- int __lock;
3000- unsigned int __futex;
3001- __extension__ unsigned long long int __total_seq;
3002- __extension__ unsigned long long int __wakeup_seq;
3003- __extension__ unsigned long long int __woken_seq;
3004- void *__mutex;
3005- unsigned int __nwaiters;
3006- unsigned int __broadcast_seq;
3007+ __extension__ union
3008+ {
3009+ __extension__ unsigned long long int __wseq;
3010+ struct {
3011+ unsigned int __low;
3012+ unsigned int __high;
3013+ } __wseq32;
3014+ };
3015+ __extension__ union
3016+ {
3017+ __extension__ unsigned long long int __g1_start;
3018+ struct {
3019+ unsigned int __low;
3020+ unsigned int __high;
3021+ } __g1_start32;
3022+ };
3023+ unsigned int __g_refs[2];
3024+ unsigned int __g_size[2];
3025+ unsigned int __g1_orig_size;
3026+ unsigned int __wrefs;
3027+ unsigned int __g_signals[2];
3028 } __data;
3029 char __size[__SIZEOF_PTHREAD_COND_T];
3030 __extension__ long long int __align;
3031diff --git a/sysdeps/nios2/nptl/bits/pthreadtypes.h b/sysdeps/nios2/nptl/bits/pthreadtypes.h
3032index 76076d0..3995e26 100644
3033--- a/sysdeps/nios2/nptl/bits/pthreadtypes.h
3034+++ b/sysdeps/nios2/nptl/bits/pthreadtypes.h
3035@@ -88,19 +88,32 @@ typedef union
3036
3037
3038 /* Data structure for conditional variable handling. The structure of
3039- the attribute type is deliberately not exposed. */
3040+ the attribute type is not exposed on purpose. */
3041 typedef union
3042 {
3043 struct
3044 {
3045- int __lock;
3046- unsigned int __futex;
3047- __extension__ unsigned long long int __total_seq;
3048- __extension__ unsigned long long int __wakeup_seq;
3049- __extension__ unsigned long long int __woken_seq;
3050- void *__mutex;
3051- unsigned int __nwaiters;
3052- unsigned int __broadcast_seq;
3053+ __extension__ union
3054+ {
3055+ __extension__ unsigned long long int __wseq;
3056+ struct {
3057+ unsigned int __low;
3058+ unsigned int __high;
3059+ } __wseq32;
3060+ };
3061+ __extension__ union
3062+ {
3063+ __extension__ unsigned long long int __g1_start;
3064+ struct {
3065+ unsigned int __low;
3066+ unsigned int __high;
3067+ } __g1_start32;
3068+ };
3069+ unsigned int __g_refs[2];
3070+ unsigned int __g_size[2];
3071+ unsigned int __g1_orig_size;
3072+ unsigned int __wrefs;
3073+ unsigned int __g_signals[2];
3074 } __data;
3075 char __size[__SIZEOF_PTHREAD_COND_T];
3076 __extension__ long long int __align;
3077diff --git a/sysdeps/nptl/internaltypes.h b/sysdeps/nptl/internaltypes.h
3078index 203c548..31e5a43 100644
3079--- a/sysdeps/nptl/internaltypes.h
3080+++ b/sysdeps/nptl/internaltypes.h
3081@@ -68,20 +68,13 @@ struct pthread_condattr
3082 {
3083 /* Combination of values:
3084
3085- Bit 0 : flag whether conditional variable will be sharable between
3086- processes.
3087-
3088- Bit 1-7: clock ID. */
3089+ Bit 0 : flag whether conditional variable will be
3090+ sharable between processes.
3091+ Bit 1-COND_CLOCK_BITS: Clock ID. COND_CLOCK_BITS is the number of bits
3092+ needed to represent the ID of the clock. */
3093 int value;
3094 };
3095-
3096-
3097-/* The __NWAITERS field is used as a counter and to house the number
3098- of bits for other purposes. COND_CLOCK_BITS is the number
3099- of bits needed to represent the ID of the clock. COND_NWAITERS_SHIFT
3100- is the number of bits reserved for other purposes like the clock. */
3101-#define COND_CLOCK_BITS 1
3102-#define COND_NWAITERS_SHIFT 1
3103+#define COND_CLOCK_BITS 1
3104
3105
3106 /* Read-write lock variable attribute data structure. */
3107diff --git a/sysdeps/nptl/pthread.h b/sysdeps/nptl/pthread.h
3108index fd0894e..c122446 100644
3109--- a/sysdeps/nptl/pthread.h
3110+++ b/sysdeps/nptl/pthread.h
3111@@ -183,7 +183,7 @@ enum
3112
3113
3114 /* Conditional variable handling. */
3115-#define PTHREAD_COND_INITIALIZER { { 0, 0, 0, 0, 0, (void *) 0, 0, 0 } }
3116+#define PTHREAD_COND_INITIALIZER { { {0}, {0}, {0, 0}, {0, 0}, 0, 0, {0, 0} } }
3117
3118
3119 /* Cleanup buffers */
3120diff --git a/sysdeps/s390/nptl/bits/pthreadtypes.h b/sysdeps/s390/nptl/bits/pthreadtypes.h
3121index 40d10fe..4e455ab 100644
3122--- a/sysdeps/s390/nptl/bits/pthreadtypes.h
3123+++ b/sysdeps/s390/nptl/bits/pthreadtypes.h
3124@@ -142,14 +142,27 @@ typedef union
3125 {
3126 struct
3127 {
3128- int __lock;
3129- unsigned int __futex;
3130- __extension__ unsigned long long int __total_seq;
3131- __extension__ unsigned long long int __wakeup_seq;
3132- __extension__ unsigned long long int __woken_seq;
3133- void *__mutex;
3134- unsigned int __nwaiters;
3135- unsigned int __broadcast_seq;
3136+ __extension__ union
3137+ {
3138+ __extension__ unsigned long long int __wseq;
3139+ struct {
3140+ unsigned int __low;
3141+ unsigned int __high;
3142+ } __wseq32;
3143+ };
3144+ __extension__ union
3145+ {
3146+ __extension__ unsigned long long int __g1_start;
3147+ struct {
3148+ unsigned int __low;
3149+ unsigned int __high;
3150+ } __g1_start32;
3151+ };
3152+ unsigned int __g_refs[2];
3153+ unsigned int __g_size[2];
3154+ unsigned int __g1_orig_size;
3155+ unsigned int __wrefs;
3156+ unsigned int __g_signals[2];
3157 } __data;
3158 char __size[__SIZEOF_PTHREAD_COND_T];
3159 __extension__ long long int __align;
3160diff --git a/sysdeps/sh/nptl/bits/pthreadtypes.h b/sysdeps/sh/nptl/bits/pthreadtypes.h
3161index 13fbd73..065dd11 100644
3162--- a/sysdeps/sh/nptl/bits/pthreadtypes.h
3163+++ b/sysdeps/sh/nptl/bits/pthreadtypes.h
3164@@ -93,14 +93,27 @@ typedef union
3165 {
3166 struct
3167 {
3168- int __lock;
3169- unsigned int __futex;
3170- __extension__ unsigned long long int __total_seq;
3171- __extension__ unsigned long long int __wakeup_seq;
3172- __extension__ unsigned long long int __woken_seq;
3173- void *__mutex;
3174- unsigned int __nwaiters;
3175- unsigned int __broadcast_seq;
3176+ __extension__ union
3177+ {
3178+ __extension__ unsigned long long int __wseq;
3179+ struct {
3180+ unsigned int __low;
3181+ unsigned int __high;
3182+ } __wseq32;
3183+ };
3184+ __extension__ union
3185+ {
3186+ __extension__ unsigned long long int __g1_start;
3187+ struct {
3188+ unsigned int __low;
3189+ unsigned int __high;
3190+ } __g1_start32;
3191+ };
3192+ unsigned int __g_refs[2];
3193+ unsigned int __g_size[2];
3194+ unsigned int __g1_orig_size;
3195+ unsigned int __wrefs;
3196+ unsigned int __g_signals[2];
3197 } __data;
3198 char __size[__SIZEOF_PTHREAD_COND_T];
3199 __extension__ long long int __align;
3200diff --git a/sysdeps/tile/nptl/bits/pthreadtypes.h b/sysdeps/tile/nptl/bits/pthreadtypes.h
3201index 7d68650..c12737f 100644
3202--- a/sysdeps/tile/nptl/bits/pthreadtypes.h
3203+++ b/sysdeps/tile/nptl/bits/pthreadtypes.h
3204@@ -122,14 +122,27 @@ typedef union
3205 {
3206 struct
3207 {
3208- int __lock;
3209- unsigned int __futex;
3210- __extension__ unsigned long long int __total_seq;
3211- __extension__ unsigned long long int __wakeup_seq;
3212- __extension__ unsigned long long int __woken_seq;
3213- void *__mutex;
3214- unsigned int __nwaiters;
3215- unsigned int __broadcast_seq;
3216+ __extension__ union
3217+ {
3218+ __extension__ unsigned long long int __wseq;
3219+ struct {
3220+ unsigned int __low;
3221+ unsigned int __high;
3222+ } __wseq32;
3223+ };
3224+ __extension__ union
3225+ {
3226+ __extension__ unsigned long long int __g1_start;
3227+ struct {
3228+ unsigned int __low;
3229+ unsigned int __high;
3230+ } __g1_start32;
3231+ };
3232+ unsigned int __g_refs[2];
3233+ unsigned int __g_size[2];
3234+ unsigned int __g1_orig_size;
3235+ unsigned int __wrefs;
3236+ unsigned int __g_signals[2];
3237 } __data;
3238 char __size[__SIZEOF_PTHREAD_COND_T];
3239 __extension__ long long int __align;
3240diff --git a/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
3241index 1a1779b..d88b045 100644
3242--- a/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
3243+++ b/sysdeps/unix/sysv/linux/alpha/bits/pthreadtypes.h
3244@@ -84,19 +84,32 @@ typedef union
3245
3246
3247 /* Data structure for conditional variable handling. The structure of
3248- the attribute type is deliberately not exposed. */
3249+ the attribute type is not exposed on purpose. */
3250 typedef union
3251 {
3252 struct
3253 {
3254- int __lock;
3255- unsigned int __futex;
3256- __extension__ unsigned long long int __total_seq;
3257- __extension__ unsigned long long int __wakeup_seq;
3258- __extension__ unsigned long long int __woken_seq;
3259- void *__mutex;
3260- unsigned int __nwaiters;
3261- unsigned int __broadcast_seq;
3262+ __extension__ union
3263+ {
3264+ __extension__ unsigned long long int __wseq;
3265+ struct {
3266+ unsigned int __low;
3267+ unsigned int __high;
3268+ } __wseq32;
3269+ };
3270+ __extension__ union
3271+ {
3272+ __extension__ unsigned long long int __g1_start;
3273+ struct {
3274+ unsigned int __low;
3275+ unsigned int __high;
3276+ } __g1_start32;
3277+ };
3278+ unsigned int __g_refs[2];
3279+ unsigned int __g_size[2];
3280+ unsigned int __g1_orig_size;
3281+ unsigned int __wrefs;
3282+ unsigned int __g_signals[2];
3283 } __data;
3284 char __size[__SIZEOF_PTHREAD_COND_T];
3285 __extension__ long long int __align;
3286diff --git a/sysdeps/unix/sysv/linux/hppa/internaltypes.h b/sysdeps/unix/sysv/linux/hppa/internaltypes.h
3287index 651ce2e..d649657 100644
3288--- a/sysdeps/unix/sysv/linux/hppa/internaltypes.h
3289+++ b/sysdeps/unix/sysv/linux/hppa/internaltypes.h
3290@@ -46,32 +46,38 @@ fails because __initializer is zero, and the structure will be used as
3291 is correctly. */
3292
3293 #define cond_compat_clear(var) \
3294-({ \
3295- int tmp = 0; \
3296- var->__data.__lock = 0; \
3297- var->__data.__futex = 0; \
3298- var->__data.__mutex = NULL; \
3299- /* Clear __initializer last, to indicate initialization is done. */ \
3300- __asm__ __volatile__ ("stw,ma %1,0(%0)" \
3301- : : "r" (&var->__data.__initializer), "r" (tmp) : "memory"); \
3302+({ \
3303+ int tmp = 0; \
3304+ var->__data.__wseq = 0; \
3305+ var->__data.__signals_sent = 0; \
3306+ var->__data.__confirmed = 0; \
3307+ var->__data.__generation = 0; \
3308+ var->__data.__mutex = NULL; \
3309+ var->__data.__quiescence_waiters = 0; \
3310+ var->__data.__clockid = 0; \
3311+ /* Clear __initializer last, to indicate initialization is done. */ \
3312+ /* This synchronizes-with the acquire load below. */ \
3313+ atomic_store_release (&var->__data.__initializer, 0); \
3314 })
3315
3316 #define cond_compat_check_and_clear(var) \
3317 ({ \
3318- int ret; \
3319- volatile int *value = &var->__data.__initializer; \
3320- if ((ret = atomic_compare_and_exchange_val_acq(value, 2, 1))) \
3321+ int v; \
3322+ int *value = &var->__data.__initializer; \
3323+ /* This synchronizes-with the release store above. */ \
3324+ while ((v = atomic_load_acquire (value)) != 0) \
3325 { \
3326- if (ret == 1) \
3327+ if (v == 1 \
3328+ /* Relaxed MO is fine; it only matters who's first. */ \
3329+ && atomic_compare_exchange_acquire_weak_relaxed (value, 1, 2)) \
3330 { \
3331- /* Initialize structure. */ \
3332+ /* We're first; initialize structure. */ \
3333 cond_compat_clear (var); \
3334+ break; \
3335 } \
3336 else \
3337- { \
3338- /* Yield until structure is initialized. */ \
3339- while (*value == 2) sched_yield (); \
3340- } \
3341+ /* Yield before we re-check initialization status. */ \
3342+ sched_yield (); \
3343 } \
3344 })
3345
3346diff --git a/sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c b/sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c
3347deleted file mode 100644
3348index ec6fd23..0000000
3349--- a/sysdeps/unix/sysv/linux/hppa/pthread_cond_timedwait.c
3350+++ /dev/null
3351@@ -1,41 +0,0 @@
3352-/* Copyright (C) 2009-2016 Free Software Foundation, Inc.
3353- This file is part of the GNU C Library.
3354- Contributed by Carlos O'Donell <carlos@codesourcery.com>, 2009.
3355-
3356- The GNU C Library is free software; you can redistribute it and/or
3357- modify it under the terms of the GNU Lesser General Public
3358- License as published by the Free Software Foundation; either
3359- version 2.1 of the License, or (at your option) any later version.
3360-
3361- The GNU C Library is distributed in the hope that it will be useful,
3362- but WITHOUT ANY WARRANTY; without even the implied warranty of
3363- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
3364- Lesser General Public License for more details.
3365-
3366- You should have received a copy of the GNU Lesser General Public
3367- License along with the GNU C Library. If not, see
3368- <http://www.gnu.org/licenses/>. */
3369-
3370-#ifndef INCLUDED_SELF
3371-# define INCLUDED_SELF
3372-# include <pthread_cond_timedwait.c>
3373-#else
3374-# include <pthread.h>
3375-# include <pthreadP.h>
3376-# include <internaltypes.h>
3377-# include <shlib-compat.h>
3378-int
3379-__pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
3380- const struct timespec *abstime)
3381-{
3382- cond_compat_check_and_clear (cond);
3383- return __pthread_cond_timedwait_internal (cond, mutex, abstime);
3384-}
3385-versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
3386- GLIBC_2_3_2);
3387-# undef versioned_symbol
3388-# define versioned_symbol(lib, local, symbol, version)
3389-# undef __pthread_cond_timedwait
3390-# define __pthread_cond_timedwait __pthread_cond_timedwait_internal
3391-# include_next <pthread_cond_timedwait.c>
3392-#endif
3393diff --git a/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c b/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c
3394index 8f02831..0611f7d 100644
3395--- a/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c
3396+++ b/sysdeps/unix/sysv/linux/hppa/pthread_cond_wait.c
3397@@ -32,9 +32,22 @@ __pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex)
3398 }
3399 versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
3400 GLIBC_2_3_2);
3401+int
3402+__pthread_cond_timedwait (cond, mutex, abstime)
3403+ pthread_cond_t *cond;
3404+ pthread_mutex_t *mutex;
3405+ const struct timespec *abstime;
3406+{
3407+ cond_compat_check_and_clear (cond);
3408+ return __pthread_cond_timedwait_internal (cond, mutex, abstime);
3409+}
3410+versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
3411+ GLIBC_2_3_2);
3412 # undef versioned_symbol
3413 # define versioned_symbol(lib, local, symbol, version)
3414 # undef __pthread_cond_wait
3415 # define __pthread_cond_wait __pthread_cond_wait_internal
3416+# undef __pthread_cond_timedwait
3417+# define __pthread_cond_timedwait __pthread_cond_timedwait_internal
3418 # include_next <pthread_cond_wait.c>
3419 #endif
3420diff --git a/sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S b/sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S
3421deleted file mode 100644
3422index f697e5b..0000000
3423--- a/sysdeps/unix/sysv/linux/i386/i686/pthread_cond_timedwait.S
3424+++ /dev/null
3425@@ -1,20 +0,0 @@
3426-/* Copyright (C) 2003-2016 Free Software Foundation, Inc.
3427- This file is part of the GNU C Library.
3428- Contributed by Ulrich Drepper <drepper@redhat.com>, 2003.
3429-
3430- The GNU C Library is free software; you can redistribute it and/or
3431- modify it under the terms of the GNU Lesser General Public
3432- License as published by the Free Software Foundation; either
3433- version 2.1 of the License, or (at your option) any later version.
3434-
3435- The GNU C Library is distributed in the hope that it will be useful,
3436- but WITHOUT ANY WARRANTY; without even the implied warranty of
3437- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
3438- Lesser General Public License for more details.
3439-
3440- You should have received a copy of the GNU Lesser General Public
3441- License along with the GNU C Library; if not, see
3442- <http://www.gnu.org/licenses/>. */
3443-
3444-#define HAVE_CMOV 1
3445-#include "../pthread_cond_timedwait.S"
3446diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S
3447deleted file mode 100644
3448index 5996688..0000000
3449--- a/sysdeps/unix/sysv/linux/i386/pthread_cond_broadcast.S
3450+++ /dev/null
3451@@ -1,241 +0,0 @@
3452-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
3453- This file is part of the GNU C Library.
3454- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
3455-
3456- The GNU C Library is free software; you can redistribute it and/or
3457- modify it under the terms of the GNU Lesser General Public
3458- License as published by the Free Software Foundation; either
3459- version 2.1 of the License, or (at your option) any later version.
3460-
3461- The GNU C Library is distributed in the hope that it will be useful,
3462- but WITHOUT ANY WARRANTY; without even the implied warranty of
3463- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
3464- Lesser General Public License for more details.
3465-
3466- You should have received a copy of the GNU Lesser General Public
3467- License along with the GNU C Library; if not, see
3468- <http://www.gnu.org/licenses/>. */
3469-
3470-#include <sysdep.h>
3471-#include <shlib-compat.h>
3472-#include <lowlevellock.h>
3473-#include <lowlevelcond.h>
3474-#include <kernel-features.h>
3475-#include <pthread-pi-defines.h>
3476-#include <pthread-errnos.h>
3477-#include <stap-probe.h>
3478-
3479- .text
3480-
3481- /* int pthread_cond_broadcast (pthread_cond_t *cond) */
3482- .globl __pthread_cond_broadcast
3483- .type __pthread_cond_broadcast, @function
3484- .align 16
3485-__pthread_cond_broadcast:
3486- cfi_startproc
3487- pushl %ebx
3488- cfi_adjust_cfa_offset(4)
3489- cfi_rel_offset(%ebx, 0)
3490- pushl %esi
3491- cfi_adjust_cfa_offset(4)
3492- cfi_rel_offset(%esi, 0)
3493- pushl %edi
3494- cfi_adjust_cfa_offset(4)
3495- cfi_rel_offset(%edi, 0)
3496- pushl %ebp
3497- cfi_adjust_cfa_offset(4)
3498- cfi_rel_offset(%ebp, 0)
3499- cfi_remember_state
3500-
3501- movl 20(%esp), %ebx
3502-
3503- LIBC_PROBE (cond_broadcast, 1, %edx)
3504-
3505- /* Get internal lock. */
3506- movl $1, %edx
3507- xorl %eax, %eax
3508- LOCK
3509-#if cond_lock == 0
3510- cmpxchgl %edx, (%ebx)
3511-#else
3512- cmpxchgl %edx, cond_lock(%ebx)
3513-#endif
3514- jnz 1f
3515-
3516-2: addl $cond_futex, %ebx
3517- movl total_seq+4-cond_futex(%ebx), %eax
3518- movl total_seq-cond_futex(%ebx), %ebp
3519- cmpl wakeup_seq+4-cond_futex(%ebx), %eax
3520- ja 3f
3521- jb 4f
3522- cmpl wakeup_seq-cond_futex(%ebx), %ebp
3523- jna 4f
3524-
3525- /* Cause all currently waiting threads to recognize they are
3526- woken up. */
3527-3: movl %ebp, wakeup_seq-cond_futex(%ebx)
3528- movl %eax, wakeup_seq-cond_futex+4(%ebx)
3529- movl %ebp, woken_seq-cond_futex(%ebx)
3530- movl %eax, woken_seq-cond_futex+4(%ebx)
3531- addl %ebp, %ebp
3532- addl $1, broadcast_seq-cond_futex(%ebx)
3533- movl %ebp, (%ebx)
3534-
3535- /* Get the address of the mutex used. */
3536- movl dep_mutex-cond_futex(%ebx), %edi
3537-
3538- /* Unlock. */
3539- LOCK
3540- subl $1, cond_lock-cond_futex(%ebx)
3541- jne 7f
3542-
3543- /* Don't use requeue for pshared condvars. */
3544-8: cmpl $-1, %edi
3545- je 9f
3546-
3547- /* Do not use requeue for pshared condvars. */
3548- testl $PS_BIT, MUTEX_KIND(%edi)
3549- jne 9f
3550-
3551- /* Requeue to a non-robust PI mutex if the PI bit is set and
3552- the robust bit is not set. */
3553- movl MUTEX_KIND(%edi), %eax
3554- andl $(ROBUST_BIT|PI_BIT), %eax
3555- cmpl $PI_BIT, %eax
3556- je 81f
3557-
3558- /* Wake up all threads. */
3559-#ifdef __ASSUME_PRIVATE_FUTEX
3560- movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
3561-#else
3562- movl %gs:PRIVATE_FUTEX, %ecx
3563- orl $FUTEX_CMP_REQUEUE, %ecx
3564-#endif
3565- movl $SYS_futex, %eax
3566- movl $0x7fffffff, %esi
3567- movl $1, %edx
3568- /* Get the address of the futex involved. */
3569-# if MUTEX_FUTEX != 0
3570- addl $MUTEX_FUTEX, %edi
3571-# endif
3572-/* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for sysenter.
3573- ENTER_KERNEL */
3574- int $0x80
3575-
3576- /* For any kind of error, which mainly is EAGAIN, we try again
3577- with WAKE. The general test also covers running on old
3578- kernels. */
3579- cmpl $0xfffff001, %eax
3580- jae 9f
3581-
3582-6: xorl %eax, %eax
3583- popl %ebp
3584- cfi_adjust_cfa_offset(-4)
3585- cfi_restore(%ebp)
3586- popl %edi
3587- cfi_adjust_cfa_offset(-4)
3588- cfi_restore(%edi)
3589- popl %esi
3590- cfi_adjust_cfa_offset(-4)
3591- cfi_restore(%esi)
3592- popl %ebx
3593- cfi_adjust_cfa_offset(-4)
3594- cfi_restore(%ebx)
3595- ret
3596-
3597- cfi_restore_state
3598-
3599-81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
3600- movl $SYS_futex, %eax
3601- movl $0x7fffffff, %esi
3602- movl $1, %edx
3603- /* Get the address of the futex involved. */
3604-# if MUTEX_FUTEX != 0
3605- addl $MUTEX_FUTEX, %edi
3606-# endif
3607- int $0x80
3608-
3609- /* For any kind of error, which mainly is EAGAIN, we try again
3610- with WAKE. The general test also covers running on old
3611- kernels. */
3612- cmpl $0xfffff001, %eax
3613- jb 6b
3614- jmp 9f
3615-
3616- /* Initial locking failed. */
3617-1:
3618-#if cond_lock == 0
3619- movl %ebx, %edx
3620-#else
3621- leal cond_lock(%ebx), %edx
3622-#endif
3623-#if (LLL_SHARED-LLL_PRIVATE) > 255
3624- xorl %ecx, %ecx
3625-#endif
3626- cmpl $-1, dep_mutex(%ebx)
3627- setne %cl
3628- subl $1, %ecx
3629- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
3630-#if LLL_PRIVATE != 0
3631- addl $LLL_PRIVATE, %ecx
3632-#endif
3633- call __lll_lock_wait
3634- jmp 2b
3635-
3636- .align 16
3637- /* Unlock. */
3638-4: LOCK
3639- subl $1, cond_lock-cond_futex(%ebx)
3640- je 6b
3641-
3642- /* Unlock in loop requires wakeup. */
3643-5: leal cond_lock-cond_futex(%ebx), %eax
3644-#if (LLL_SHARED-LLL_PRIVATE) > 255
3645- xorl %ecx, %ecx
3646-#endif
3647- cmpl $-1, dep_mutex-cond_futex(%ebx)
3648- setne %cl
3649- subl $1, %ecx
3650- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
3651-#if LLL_PRIVATE != 0
3652- addl $LLL_PRIVATE, %ecx
3653-#endif
3654- call __lll_unlock_wake
3655- jmp 6b
3656-
3657- /* Unlock in loop requires wakeup. */
3658-7: leal cond_lock-cond_futex(%ebx), %eax
3659-#if (LLL_SHARED-LLL_PRIVATE) > 255
3660- xorl %ecx, %ecx
3661-#endif
3662- cmpl $-1, dep_mutex-cond_futex(%ebx)
3663- setne %cl
3664- subl $1, %ecx
3665- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
3666-#if LLL_PRIVATE != 0
3667- addl $LLL_PRIVATE, %ecx
3668-#endif
3669- call __lll_unlock_wake
3670- jmp 8b
3671-
3672-9: /* The futex requeue functionality is not available. */
3673- movl $0x7fffffff, %edx
3674-#if FUTEX_PRIVATE_FLAG > 255
3675- xorl %ecx, %ecx
3676-#endif
3677- cmpl $-1, dep_mutex-cond_futex(%ebx)
3678- sete %cl
3679- subl $1, %ecx
3680-#ifdef __ASSUME_PRIVATE_FUTEX
3681- andl $FUTEX_PRIVATE_FLAG, %ecx
3682-#else
3683- andl %gs:PRIVATE_FUTEX, %ecx
3684-#endif
3685- addl $FUTEX_WAKE, %ecx
3686- movl $SYS_futex, %eax
3687- ENTER_KERNEL
3688- jmp 6b
3689- cfi_endproc
3690- .size __pthread_cond_broadcast, .-__pthread_cond_broadcast
3691-versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
3692- GLIBC_2_3_2)
3693diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S
3694deleted file mode 100644
3695index 0038775..0000000
3696--- a/sysdeps/unix/sysv/linux/i386/pthread_cond_signal.S
3697+++ /dev/null
3698@@ -1,216 +0,0 @@
3699-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
3700- This file is part of the GNU C Library.
3701- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
3702-
3703- The GNU C Library is free software; you can redistribute it and/or
3704- modify it under the terms of the GNU Lesser General Public
3705- License as published by the Free Software Foundation; either
3706- version 2.1 of the License, or (at your option) any later version.
3707-
3708- The GNU C Library is distributed in the hope that it will be useful,
3709- but WITHOUT ANY WARRANTY; without even the implied warranty of
3710- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
3711- Lesser General Public License for more details.
3712-
3713- You should have received a copy of the GNU Lesser General Public
3714- License along with the GNU C Library; if not, see
3715- <http://www.gnu.org/licenses/>. */
3716-
3717-#include <sysdep.h>
3718-#include <shlib-compat.h>
3719-#include <lowlevellock.h>
3720-#include <lowlevelcond.h>
3721-#include <kernel-features.h>
3722-#include <pthread-pi-defines.h>
3723-#include <pthread-errnos.h>
3724-#include <stap-probe.h>
3725-
3726- .text
3727-
3728- /* int pthread_cond_signal (pthread_cond_t *cond) */
3729- .globl __pthread_cond_signal
3730- .type __pthread_cond_signal, @function
3731- .align 16
3732-__pthread_cond_signal:
3733-
3734- cfi_startproc
3735- pushl %ebx
3736- cfi_adjust_cfa_offset(4)
3737- cfi_rel_offset(%ebx, 0)
3738- pushl %edi
3739- cfi_adjust_cfa_offset(4)
3740- cfi_rel_offset(%edi, 0)
3741- cfi_remember_state
3742-
3743- movl 12(%esp), %edi
3744-
3745- LIBC_PROBE (cond_signal, 1, %edi)
3746-
3747- /* Get internal lock. */
3748- movl $1, %edx
3749- xorl %eax, %eax
3750- LOCK
3751-#if cond_lock == 0
3752- cmpxchgl %edx, (%edi)
3753-#else
3754- cmpxchgl %edx, cond_lock(%edi)
3755-#endif
3756- jnz 1f
3757-
3758-2: leal cond_futex(%edi), %ebx
3759- movl total_seq+4(%edi), %eax
3760- movl total_seq(%edi), %ecx
3761- cmpl wakeup_seq+4(%edi), %eax
3762-#if cond_lock != 0
3763- /* Must use leal to preserve the flags. */
3764- leal cond_lock(%edi), %edi
3765-#endif
3766- ja 3f
3767- jb 4f
3768- cmpl wakeup_seq-cond_futex(%ebx), %ecx
3769- jbe 4f
3770-
3771- /* Bump the wakeup number. */
3772-3: addl $1, wakeup_seq-cond_futex(%ebx)
3773- adcl $0, wakeup_seq-cond_futex+4(%ebx)
3774- addl $1, (%ebx)
3775-
3776- /* Wake up one thread. */
3777- pushl %esi
3778- cfi_adjust_cfa_offset(4)
3779- cfi_rel_offset(%esi, 0)
3780- pushl %ebp
3781- cfi_adjust_cfa_offset(4)
3782- cfi_rel_offset(%ebp, 0)
3783-
3784-#if FUTEX_PRIVATE_FLAG > 255
3785- xorl %ecx, %ecx
3786-#endif
3787- cmpl $-1, dep_mutex-cond_futex(%ebx)
3788- sete %cl
3789- je 8f
3790-
3791- movl dep_mutex-cond_futex(%ebx), %edx
3792- /* Requeue to a non-robust PI mutex if the PI bit is set and
3793- the robust bit is not set. */
3794- movl MUTEX_KIND(%edx), %eax
3795- andl $(ROBUST_BIT|PI_BIT), %eax
3796- cmpl $PI_BIT, %eax
3797- je 9f
3798-
3799-8: subl $1, %ecx
3800-#ifdef __ASSUME_PRIVATE_FUTEX
3801- andl $FUTEX_PRIVATE_FLAG, %ecx
3802-#else
3803- andl %gs:PRIVATE_FUTEX, %ecx
3804-#endif
3805- addl $FUTEX_WAKE_OP, %ecx
3806- movl $SYS_futex, %eax
3807- movl $1, %edx
3808- movl $1, %esi
3809- movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %ebp
3810- /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
3811- sysenter.
3812- ENTER_KERNEL */
3813- int $0x80
3814- popl %ebp
3815- cfi_adjust_cfa_offset(-4)
3816- cfi_restore(%ebp)
3817- popl %esi
3818- cfi_adjust_cfa_offset(-4)
3819- cfi_restore(%esi)
3820-
3821- /* For any kind of error, we try again with WAKE.
3822- The general test also covers running on old kernels. */
3823- cmpl $-4095, %eax
3824- jae 7f
3825-
3826-6: xorl %eax, %eax
3827- popl %edi
3828- cfi_adjust_cfa_offset(-4)
3829- cfi_restore(%edi)
3830- popl %ebx
3831- cfi_adjust_cfa_offset(-4)
3832- cfi_restore(%ebx)
3833- ret
3834-
3835- cfi_restore_state
3836-
3837-9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
3838- movl $SYS_futex, %eax
3839- movl $1, %edx
3840- xorl %esi, %esi
3841- movl dep_mutex-cond_futex(%ebx), %edi
3842- movl (%ebx), %ebp
3843- /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
3844- sysenter.
3845- ENTER_KERNEL */
3846- int $0x80
3847- popl %ebp
3848- popl %esi
3849-
3850- leal -cond_futex(%ebx), %edi
3851-
3852- /* For any kind of error, we try again with WAKE.
3853- The general test also covers running on old kernels. */
3854- cmpl $-4095, %eax
3855- jb 4f
3856-
3857-7:
3858-#ifdef __ASSUME_PRIVATE_FUTEX
3859- andl $FUTEX_PRIVATE_FLAG, %ecx
3860-#else
3861- andl %gs:PRIVATE_FUTEX, %ecx
3862-#endif
3863- orl $FUTEX_WAKE, %ecx
3864-
3865- movl $SYS_futex, %eax
3866- /* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
3867- movl $1, %edx */
3868- ENTER_KERNEL
3869-
3870- /* Unlock. Note that at this point %edi always points to
3871- cond_lock. */
3872-4: LOCK
3873- subl $1, (%edi)
3874- je 6b
3875-
3876- /* Unlock in loop requires wakeup. */
3877-5: movl %edi, %eax
3878-#if (LLL_SHARED-LLL_PRIVATE) > 255
3879- xorl %ecx, %ecx
3880-#endif
3881- cmpl $-1, dep_mutex-cond_futex(%ebx)
3882- setne %cl
3883- subl $1, %ecx
3884- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
3885-#if LLL_PRIVATE != 0
3886- addl $LLL_PRIVATE, %ecx
3887-#endif
3888- call __lll_unlock_wake
3889- jmp 6b
3890-
3891- /* Initial locking failed. */
3892-1:
3893-#if cond_lock == 0
3894- movl %edi, %edx
3895-#else
3896- leal cond_lock(%edi), %edx
3897-#endif
3898-#if (LLL_SHARED-LLL_PRIVATE) > 255
3899- xorl %ecx, %ecx
3900-#endif
3901- cmpl $-1, dep_mutex(%edi)
3902- setne %cl
3903- subl $1, %ecx
3904- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
3905-#if LLL_PRIVATE != 0
3906- addl $LLL_PRIVATE, %ecx
3907-#endif
3908- call __lll_lock_wait
3909- jmp 2b
3910-
3911- cfi_endproc
3912- .size __pthread_cond_signal, .-__pthread_cond_signal
3913-versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
3914- GLIBC_2_3_2)
3915diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S
3916deleted file mode 100644
3917index 6256376..0000000
3918--- a/sysdeps/unix/sysv/linux/i386/pthread_cond_timedwait.S
3919+++ /dev/null
3920@@ -1,974 +0,0 @@
3921-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
3922- This file is part of the GNU C Library.
3923- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
3924-
3925- The GNU C Library is free software; you can redistribute it and/or
3926- modify it under the terms of the GNU Lesser General Public
3927- License as published by the Free Software Foundation; either
3928- version 2.1 of the License, or (at your option) any later version.
3929-
3930- The GNU C Library is distributed in the hope that it will be useful,
3931- but WITHOUT ANY WARRANTY; without even the implied warranty of
3932- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
3933- Lesser General Public License for more details.
3934-
3935- You should have received a copy of the GNU Lesser General Public
3936- License along with the GNU C Library; if not, see
3937- <http://www.gnu.org/licenses/>. */
3938-
3939-#include <sysdep.h>
3940-#include <shlib-compat.h>
3941-#include <lowlevellock.h>
3942-#include <lowlevelcond.h>
3943-#include <pthread-errnos.h>
3944-#include <pthread-pi-defines.h>
3945-#include <kernel-features.h>
3946-#include <stap-probe.h>
3947-
3948- .text
3949-
3950-/* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
3951- const struct timespec *abstime) */
3952- .globl __pthread_cond_timedwait
3953- .type __pthread_cond_timedwait, @function
3954- .align 16
3955-__pthread_cond_timedwait:
3956-.LSTARTCODE:
3957- cfi_startproc
3958-#ifdef SHARED
3959- cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
3960- DW.ref.__gcc_personality_v0)
3961- cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
3962-#else
3963- cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
3964- cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
3965-#endif
3966-
3967- pushl %ebp
3968- cfi_adjust_cfa_offset(4)
3969- cfi_rel_offset(%ebp, 0)
3970- pushl %edi
3971- cfi_adjust_cfa_offset(4)
3972- cfi_rel_offset(%edi, 0)
3973- pushl %esi
3974- cfi_adjust_cfa_offset(4)
3975- cfi_rel_offset(%esi, 0)
3976- pushl %ebx
3977- cfi_adjust_cfa_offset(4)
3978- cfi_rel_offset(%ebx, 0)
3979-
3980- movl 20(%esp), %ebx
3981- movl 28(%esp), %ebp
3982-
3983- LIBC_PROBE (cond_timedwait, 3, %ebx, 24(%esp), %ebp)
3984-
3985- cmpl $1000000000, 4(%ebp)
3986- movl $EINVAL, %eax
3987- jae 18f
3988-
3989- /* Stack frame:
3990-
3991- esp + 32
3992- +--------------------------+
3993- esp + 24 | timeout value |
3994- +--------------------------+
3995- esp + 20 | futex pointer |
3996- +--------------------------+
3997- esp + 16 | pi-requeued flag |
3998- +--------------------------+
3999- esp + 12 | old broadcast_seq value |
4000- +--------------------------+
4001- esp + 4 | old wake_seq value |
4002- +--------------------------+
4003- esp + 0 | old cancellation mode |
4004- +--------------------------+
4005- */
4006-
4007-#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
4008-# ifdef PIC
4009- LOAD_PIC_REG (cx)
4010- cmpl $0, __have_futex_clock_realtime@GOTOFF(%ecx)
4011-# else
4012- cmpl $0, __have_futex_clock_realtime
4013-# endif
4014- je .Lreltmo
4015-#endif
4016-
4017- /* Get internal lock. */
4018- movl $1, %edx
4019- xorl %eax, %eax
4020- LOCK
4021-#if cond_lock == 0
4022- cmpxchgl %edx, (%ebx)
4023-#else
4024- cmpxchgl %edx, cond_lock(%ebx)
4025-#endif
4026- jnz 1f
4027-
4028- /* Store the reference to the mutex. If there is already a
4029- different value in there this is a bad user bug. */
4030-2: cmpl $-1, dep_mutex(%ebx)
4031- movl 24(%esp), %eax
4032- je 17f
4033- movl %eax, dep_mutex(%ebx)
4034-
4035- /* Unlock the mutex. */
4036-17: xorl %edx, %edx
4037- call __pthread_mutex_unlock_usercnt
4038-
4039- testl %eax, %eax
4040- jne 16f
4041-
4042- addl $1, total_seq(%ebx)
4043- adcl $0, total_seq+4(%ebx)
4044- addl $1, cond_futex(%ebx)
4045- addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
4046-
4047-#ifdef __ASSUME_FUTEX_CLOCK_REALTIME
4048-# define FRAME_SIZE 24
4049-#else
4050-# define FRAME_SIZE 32
4051-#endif
4052- subl $FRAME_SIZE, %esp
4053- cfi_adjust_cfa_offset(FRAME_SIZE)
4054- cfi_remember_state
4055-
4056- /* Get and store current wakeup_seq value. */
4057- movl wakeup_seq(%ebx), %edi
4058- movl wakeup_seq+4(%ebx), %edx
4059- movl broadcast_seq(%ebx), %eax
4060- movl %edi, 4(%esp)
4061- movl %edx, 8(%esp)
4062- movl %eax, 12(%esp)
4063-
4064- /* Reset the pi-requeued flag. */
4065- movl $0, 16(%esp)
4066-
4067- cmpl $0, (%ebp)
4068- movl $-ETIMEDOUT, %esi
4069- js 6f
4070-
4071-8: movl cond_futex(%ebx), %edi
4072- movl %edi, 20(%esp)
4073-
4074- /* Unlock. */
4075- LOCK
4076-#if cond_lock == 0
4077- subl $1, (%ebx)
4078-#else
4079- subl $1, cond_lock(%ebx)
4080-#endif
4081- jne 3f
4082-
4083-.LcleanupSTART:
4084-4: call __pthread_enable_asynccancel
4085- movl %eax, (%esp)
4086-
4087- leal (%ebp), %esi
4088-#if FUTEX_PRIVATE_FLAG > 255
4089- xorl %ecx, %ecx
4090-#endif
4091- cmpl $-1, dep_mutex(%ebx)
4092- sete %cl
4093- je 40f
4094-
4095- movl dep_mutex(%ebx), %edi
4096- /* Requeue to a non-robust PI mutex if the PI bit is set and
4097- the robust bit is not set. */
4098- movl MUTEX_KIND(%edi), %eax
4099- andl $(ROBUST_BIT|PI_BIT), %eax
4100- cmpl $PI_BIT, %eax
4101- jne 40f
4102-
4103- movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
4104- /* The following only works like this because we only support
4105- two clocks, represented using a single bit. */
4106- testl $1, cond_nwaiters(%ebx)
4107- /* XXX Need to implement using sete instead of a jump. */
4108- jne 42f
4109- orl $FUTEX_CLOCK_REALTIME, %ecx
4110-
4111-42: movl 20(%esp), %edx
4112- addl $cond_futex, %ebx
4113-.Ladd_cond_futex_pi:
4114- movl $SYS_futex, %eax
4115- ENTER_KERNEL
4116- subl $cond_futex, %ebx
4117-.Lsub_cond_futex_pi:
4118- movl %eax, %esi
4119- /* Set the pi-requeued flag only if the kernel has returned 0. The
4120- kernel does not hold the mutex on ETIMEDOUT or any other error. */
4121- cmpl $0, %eax
4122- sete 16(%esp)
4123- je 41f
4124-
4125- /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
4126- successfully, it has already locked the mutex for us and the
4127- pi_flag (16(%esp)) is set to denote that fact. However, if another
4128- thread changed the futex value before we entered the wait, the
4129- syscall may return an EAGAIN and the mutex is not locked. We go
4130- ahead with a success anyway since later we look at the pi_flag to
4131- decide if we got the mutex or not. The sequence numbers then make
4132- sure that only one of the threads actually wake up. We retry using
4133- normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
4134- and PI futexes don't mix.
4135-
4136- Note that we don't check for EAGAIN specifically; we assume that the
4137- only other error the futex function could return is EAGAIN (barring
4138- the ETIMEOUT of course, for the timeout case in futex) since
4139- anything else would mean an error in our function. It is too
4140- expensive to do that check for every call (which is quite common in
4141- case of a large number of threads), so it has been skipped. */
4142- cmpl $-ENOSYS, %eax
4143- jne 41f
4144- xorl %ecx, %ecx
4145-
4146-40: subl $1, %ecx
4147- movl $0, 16(%esp)
4148-#ifdef __ASSUME_PRIVATE_FUTEX
4149- andl $FUTEX_PRIVATE_FLAG, %ecx
4150-#else
4151- andl %gs:PRIVATE_FUTEX, %ecx
4152-#endif
4153- addl $FUTEX_WAIT_BITSET, %ecx
4154- /* The following only works like this because we only support
4155- two clocks, represented using a single bit. */
4156- testl $1, cond_nwaiters(%ebx)
4157- jne 30f
4158- orl $FUTEX_CLOCK_REALTIME, %ecx
4159-30:
4160- movl 20(%esp), %edx
4161- movl $0xffffffff, %ebp
4162- addl $cond_futex, %ebx
4163-.Ladd_cond_futex:
4164- movl $SYS_futex, %eax
4165- ENTER_KERNEL
4166- subl $cond_futex, %ebx
4167-.Lsub_cond_futex:
4168- movl 28+FRAME_SIZE(%esp), %ebp
4169- movl %eax, %esi
4170-
4171-41: movl (%esp), %eax
4172- call __pthread_disable_asynccancel
4173-.LcleanupEND:
4174-
4175- /* Lock. */
4176- movl $1, %edx
4177- xorl %eax, %eax
4178- LOCK
4179-#if cond_lock == 0
4180- cmpxchgl %edx, (%ebx)
4181-#else
4182- cmpxchgl %edx, cond_lock(%ebx)
4183-#endif
4184- jnz 5f
4185-
4186-6: movl broadcast_seq(%ebx), %eax
4187- cmpl 12(%esp), %eax
4188- jne 23f
4189-
4190- movl woken_seq(%ebx), %eax
4191- movl woken_seq+4(%ebx), %ecx
4192-
4193- movl wakeup_seq(%ebx), %edi
4194- movl wakeup_seq+4(%ebx), %edx
4195-
4196- cmpl 8(%esp), %edx
4197- jne 7f
4198- cmpl 4(%esp), %edi
4199- je 15f
4200-
4201-7: cmpl %ecx, %edx
4202- jne 9f
4203- cmp %eax, %edi
4204- jne 9f
4205-
4206-15: cmpl $-ETIMEDOUT, %esi
4207- je 28f
4208-
4209- /* We need to go back to futex_wait. If we're using requeue_pi, then
4210- release the mutex we had acquired and go back. */
4211- movl 16(%esp), %edx
4212- test %edx, %edx
4213- jz 8b
4214-
4215- /* Adjust the mutex values first and then unlock it. The unlock
4216- should always succeed or else the kernel did not lock the mutex
4217- correctly. */
4218- movl dep_mutex(%ebx), %eax
4219- call __pthread_mutex_cond_lock_adjust
4220- movl dep_mutex(%ebx), %eax
4221- xorl %edx, %edx
4222- call __pthread_mutex_unlock_usercnt
4223- jmp 8b
4224-
4225-28: addl $1, wakeup_seq(%ebx)
4226- adcl $0, wakeup_seq+4(%ebx)
4227- addl $1, cond_futex(%ebx)
4228- movl $ETIMEDOUT, %esi
4229- jmp 14f
4230-
4231-23: xorl %esi, %esi
4232- jmp 24f
4233-
4234-9: xorl %esi, %esi
4235-14: addl $1, woken_seq(%ebx)
4236- adcl $0, woken_seq+4(%ebx)
4237-
4238-24: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
4239-
4240- /* Wake up a thread which wants to destroy the condvar object. */
4241- movl total_seq(%ebx), %eax
4242- andl total_seq+4(%ebx), %eax
4243- cmpl $0xffffffff, %eax
4244- jne 25f
4245- movl cond_nwaiters(%ebx), %eax
4246- andl $~((1 << nwaiters_shift) - 1), %eax
4247- jne 25f
4248-
4249- addl $cond_nwaiters, %ebx
4250- movl $SYS_futex, %eax
4251-#if FUTEX_PRIVATE_FLAG > 255
4252- xorl %ecx, %ecx
4253-#endif
4254- cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
4255- sete %cl
4256- subl $1, %ecx
4257-#ifdef __ASSUME_PRIVATE_FUTEX
4258- andl $FUTEX_PRIVATE_FLAG, %ecx
4259-#else
4260- andl %gs:PRIVATE_FUTEX, %ecx
4261-#endif
4262- addl $FUTEX_WAKE, %ecx
4263- movl $1, %edx
4264- ENTER_KERNEL
4265- subl $cond_nwaiters, %ebx
4266-
4267-25: LOCK
4268-#if cond_lock == 0
4269- subl $1, (%ebx)
4270-#else
4271- subl $1, cond_lock(%ebx)
4272-#endif
4273- jne 10f
4274-
4275-11: movl 24+FRAME_SIZE(%esp), %eax
4276- /* With requeue_pi, the mutex lock is held in the kernel. */
4277- movl 16(%esp), %ecx
4278- testl %ecx, %ecx
4279- jnz 27f
4280-
4281- call __pthread_mutex_cond_lock
4282-26: addl $FRAME_SIZE, %esp
4283- cfi_adjust_cfa_offset(-FRAME_SIZE)
4284-
4285- /* We return the result of the mutex_lock operation if it failed. */
4286- testl %eax, %eax
4287-#ifdef HAVE_CMOV
4288- cmovel %esi, %eax
4289-#else
4290- jne 22f
4291- movl %esi, %eax
4292-22:
4293-#endif
4294-
4295-18: popl %ebx
4296- cfi_adjust_cfa_offset(-4)
4297- cfi_restore(%ebx)
4298- popl %esi
4299- cfi_adjust_cfa_offset(-4)
4300- cfi_restore(%esi)
4301- popl %edi
4302- cfi_adjust_cfa_offset(-4)
4303- cfi_restore(%edi)
4304- popl %ebp
4305- cfi_adjust_cfa_offset(-4)
4306- cfi_restore(%ebp)
4307-
4308- ret
4309-
4310- cfi_restore_state
4311-
4312-27: call __pthread_mutex_cond_lock_adjust
4313- xorl %eax, %eax
4314- jmp 26b
4315-
4316- cfi_adjust_cfa_offset(-FRAME_SIZE);
4317- /* Initial locking failed. */
4318-1:
4319-#if cond_lock == 0
4320- movl %ebx, %edx
4321-#else
4322- leal cond_lock(%ebx), %edx
4323-#endif
4324-#if (LLL_SHARED-LLL_PRIVATE) > 255
4325- xorl %ecx, %ecx
4326-#endif
4327- cmpl $-1, dep_mutex(%ebx)
4328- setne %cl
4329- subl $1, %ecx
4330- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4331-#if LLL_PRIVATE != 0
4332- addl $LLL_PRIVATE, %ecx
4333-#endif
4334- call __lll_lock_wait
4335- jmp 2b
4336-
4337- /* The initial unlocking of the mutex failed. */
4338-16:
4339- LOCK
4340-#if cond_lock == 0
4341- subl $1, (%ebx)
4342-#else
4343- subl $1, cond_lock(%ebx)
4344-#endif
4345- jne 18b
4346-
4347- movl %eax, %esi
4348-#if cond_lock == 0
4349- movl %ebx, %eax
4350-#else
4351- leal cond_lock(%ebx), %eax
4352-#endif
4353-#if (LLL_SHARED-LLL_PRIVATE) > 255
4354- xorl %ecx, %ecx
4355-#endif
4356- cmpl $-1, dep_mutex(%ebx)
4357- setne %cl
4358- subl $1, %ecx
4359- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4360-#if LLL_PRIVATE != 0
4361- addl $LLL_PRIVATE, %ecx
4362-#endif
4363- call __lll_unlock_wake
4364-
4365- movl %esi, %eax
4366- jmp 18b
4367-
4368- cfi_adjust_cfa_offset(FRAME_SIZE)
4369-
4370- /* Unlock in loop requires wakeup. */
4371-3:
4372-#if cond_lock == 0
4373- movl %ebx, %eax
4374-#else
4375- leal cond_lock(%ebx), %eax
4376-#endif
4377-#if (LLL_SHARED-LLL_PRIVATE) > 255
4378- xorl %ecx, %ecx
4379-#endif
4380- cmpl $-1, dep_mutex(%ebx)
4381- setne %cl
4382- subl $1, %ecx
4383- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4384-#if LLL_PRIVATE != 0
4385- addl $LLL_PRIVATE, %ecx
4386-#endif
4387- call __lll_unlock_wake
4388- jmp 4b
4389-
4390- /* Locking in loop failed. */
4391-5:
4392-#if cond_lock == 0
4393- movl %ebx, %edx
4394-#else
4395- leal cond_lock(%ebx), %edx
4396-#endif
4397-#if (LLL_SHARED-LLL_PRIVATE) > 255
4398- xorl %ecx, %ecx
4399-#endif
4400- cmpl $-1, dep_mutex(%ebx)
4401- setne %cl
4402- subl $1, %ecx
4403- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4404-#if LLL_PRIVATE != 0
4405- addl $LLL_PRIVATE, %ecx
4406-#endif
4407- call __lll_lock_wait
4408- jmp 6b
4409-
4410- /* Unlock after loop requires wakeup. */
4411-10:
4412-#if cond_lock == 0
4413- movl %ebx, %eax
4414-#else
4415- leal cond_lock(%ebx), %eax
4416-#endif
4417-#if (LLL_SHARED-LLL_PRIVATE) > 255
4418- xorl %ecx, %ecx
4419-#endif
4420- cmpl $-1, dep_mutex(%ebx)
4421- setne %cl
4422- subl $1, %ecx
4423- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4424-#if LLL_PRIVATE != 0
4425- addl $LLL_PRIVATE, %ecx
4426-#endif
4427- call __lll_unlock_wake
4428- jmp 11b
4429-
4430-#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
4431- cfi_adjust_cfa_offset(-FRAME_SIZE)
4432-.Lreltmo:
4433- /* Get internal lock. */
4434- movl $1, %edx
4435- xorl %eax, %eax
4436- LOCK
4437-# if cond_lock == 0
4438- cmpxchgl %edx, (%ebx)
4439-# else
4440- cmpxchgl %edx, cond_lock(%ebx)
4441-# endif
4442- jnz 101f
4443-
4444- /* Store the reference to the mutex. If there is already a
4445- different value in there this is a bad user bug. */
4446-102: cmpl $-1, dep_mutex(%ebx)
4447- movl 24(%esp), %eax
4448- je 117f
4449- movl %eax, dep_mutex(%ebx)
4450-
4451- /* Unlock the mutex. */
4452-117: xorl %edx, %edx
4453- call __pthread_mutex_unlock_usercnt
4454-
4455- testl %eax, %eax
4456- jne 16b
4457-
4458- addl $1, total_seq(%ebx)
4459- adcl $0, total_seq+4(%ebx)
4460- addl $1, cond_futex(%ebx)
4461- addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
4462-
4463- subl $FRAME_SIZE, %esp
4464- cfi_adjust_cfa_offset(FRAME_SIZE)
4465-
4466- /* Get and store current wakeup_seq value. */
4467- movl wakeup_seq(%ebx), %edi
4468- movl wakeup_seq+4(%ebx), %edx
4469- movl broadcast_seq(%ebx), %eax
4470- movl %edi, 4(%esp)
4471- movl %edx, 8(%esp)
4472- movl %eax, 12(%esp)
4473-
4474- /* Reset the pi-requeued flag. */
4475- movl $0, 16(%esp)
4476-
4477- /* Get the current time. */
4478-108: movl %ebx, %edx
4479-# ifdef __NR_clock_gettime
4480- /* Get the clock number. */
4481- movl cond_nwaiters(%ebx), %ebx
4482- andl $((1 << nwaiters_shift) - 1), %ebx
4483- /* Only clocks 0 and 1 are allowed so far. Both are handled in the
4484- kernel. */
4485- leal 24(%esp), %ecx
4486- movl $__NR_clock_gettime, %eax
4487- ENTER_KERNEL
4488- movl %edx, %ebx
4489-
4490- /* Compute relative timeout. */
4491- movl (%ebp), %ecx
4492- movl 4(%ebp), %edx
4493- subl 24(%esp), %ecx
4494- subl 28(%esp), %edx
4495-# else
4496- /* Get the current time. */
4497- leal 24(%esp), %ebx
4498- xorl %ecx, %ecx
4499- movl $__NR_gettimeofday, %eax
4500- ENTER_KERNEL
4501- movl %edx, %ebx
4502-
4503- /* Compute relative timeout. */
4504- movl 28(%esp), %eax
4505- movl $1000, %edx
4506- mul %edx /* Milli seconds to nano seconds. */
4507- movl (%ebp), %ecx
4508- movl 4(%ebp), %edx
4509- subl 24(%esp), %ecx
4510- subl %eax, %edx
4511-# endif
4512- jns 112f
4513- addl $1000000000, %edx
4514- subl $1, %ecx
4515-112: testl %ecx, %ecx
4516- movl $-ETIMEDOUT, %esi
4517- js 106f
4518-
4519- /* Store relative timeout. */
4520-121: movl %ecx, 24(%esp)
4521- movl %edx, 28(%esp)
4522-
4523- movl cond_futex(%ebx), %edi
4524- movl %edi, 20(%esp)
4525-
4526- /* Unlock. */
4527- LOCK
4528-# if cond_lock == 0
4529- subl $1, (%ebx)
4530-# else
4531- subl $1, cond_lock(%ebx)
4532-# endif
4533- jne 103f
4534-
4535-.LcleanupSTART2:
4536-104: call __pthread_enable_asynccancel
4537- movl %eax, (%esp)
4538-
4539- leal 24(%esp), %esi
4540-# if FUTEX_PRIVATE_FLAG > 255
4541- xorl %ecx, %ecx
4542-# endif
4543- cmpl $-1, dep_mutex(%ebx)
4544- sete %cl
4545- subl $1, %ecx
4546-# ifdef __ASSUME_PRIVATE_FUTEX
4547- andl $FUTEX_PRIVATE_FLAG, %ecx
4548-# else
4549- andl %gs:PRIVATE_FUTEX, %ecx
4550-# endif
4551-# if FUTEX_WAIT != 0
4552- addl $FUTEX_WAIT, %ecx
4553-# endif
4554- movl 20(%esp), %edx
4555- addl $cond_futex, %ebx
4556-.Ladd_cond_futex2:
4557- movl $SYS_futex, %eax
4558- ENTER_KERNEL
4559- subl $cond_futex, %ebx
4560-.Lsub_cond_futex2:
4561- movl %eax, %esi
4562-
4563-141: movl (%esp), %eax
4564- call __pthread_disable_asynccancel
4565-.LcleanupEND2:
4566-
4567-
4568- /* Lock. */
4569- movl $1, %edx
4570- xorl %eax, %eax
4571- LOCK
4572-# if cond_lock == 0
4573- cmpxchgl %edx, (%ebx)
4574-# else
4575- cmpxchgl %edx, cond_lock(%ebx)
4576-# endif
4577- jnz 105f
4578-
4579-106: movl broadcast_seq(%ebx), %eax
4580- cmpl 12(%esp), %eax
4581- jne 23b
4582-
4583- movl woken_seq(%ebx), %eax
4584- movl woken_seq+4(%ebx), %ecx
4585-
4586- movl wakeup_seq(%ebx), %edi
4587- movl wakeup_seq+4(%ebx), %edx
4588-
4589- cmpl 8(%esp), %edx
4590- jne 107f
4591- cmpl 4(%esp), %edi
4592- je 115f
4593-
4594-107: cmpl %ecx, %edx
4595- jne 9b
4596- cmp %eax, %edi
4597- jne 9b
4598-
4599-115: cmpl $-ETIMEDOUT, %esi
4600- je 28b
4601-
4602- jmp 8b
4603-
4604- cfi_adjust_cfa_offset(-FRAME_SIZE)
4605- /* Initial locking failed. */
4606-101:
4607-# if cond_lock == 0
4608- movl %ebx, %edx
4609-# else
4610- leal cond_lock(%ebx), %edx
4611-# endif
4612-# if (LLL_SHARED-LLL_PRIVATE) > 255
4613- xorl %ecx, %ecx
4614-# endif
4615- cmpl $-1, dep_mutex(%ebx)
4616- setne %cl
4617- subl $1, %ecx
4618- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4619-# if LLL_PRIVATE != 0
4620- addl $LLL_PRIVATE, %ecx
4621-# endif
4622- call __lll_lock_wait
4623- jmp 102b
4624-
4625- cfi_adjust_cfa_offset(FRAME_SIZE)
4626-
4627- /* Unlock in loop requires wakeup. */
4628-103:
4629-# if cond_lock == 0
4630- movl %ebx, %eax
4631-# else
4632- leal cond_lock(%ebx), %eax
4633-# endif
4634-# if (LLL_SHARED-LLL_PRIVATE) > 255
4635- xorl %ecx, %ecx
4636-# endif
4637- cmpl $-1, dep_mutex(%ebx)
4638- setne %cl
4639- subl $1, %ecx
4640- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4641-# if LLL_PRIVATE != 0
4642- addl $LLL_PRIVATE, %ecx
4643-# endif
4644- call __lll_unlock_wake
4645- jmp 104b
4646-
4647- /* Locking in loop failed. */
4648-105:
4649-# if cond_lock == 0
4650- movl %ebx, %edx
4651-# else
4652- leal cond_lock(%ebx), %edx
4653-# endif
4654-# if (LLL_SHARED-LLL_PRIVATE) > 255
4655- xorl %ecx, %ecx
4656-# endif
4657- cmpl $-1, dep_mutex(%ebx)
4658- setne %cl
4659- subl $1, %ecx
4660- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4661-# if LLL_PRIVATE != 0
4662- addl $LLL_PRIVATE, %ecx
4663-# endif
4664- call __lll_lock_wait
4665- jmp 106b
4666-#endif
4667-
4668- .size __pthread_cond_timedwait, .-__pthread_cond_timedwait
4669-versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
4670- GLIBC_2_3_2)
4671-
4672-
4673- .type __condvar_tw_cleanup2, @function
4674-__condvar_tw_cleanup2:
4675- subl $cond_futex, %ebx
4676- .size __condvar_tw_cleanup2, .-__condvar_tw_cleanup2
4677- .type __condvar_tw_cleanup, @function
4678-__condvar_tw_cleanup:
4679- movl %eax, %esi
4680-
4681- /* Get internal lock. */
4682- movl $1, %edx
4683- xorl %eax, %eax
4684- LOCK
4685-#if cond_lock == 0
4686- cmpxchgl %edx, (%ebx)
4687-#else
4688- cmpxchgl %edx, cond_lock(%ebx)
4689-#endif
4690- jz 1f
4691-
4692-#if cond_lock == 0
4693- movl %ebx, %edx
4694-#else
4695- leal cond_lock(%ebx), %edx
4696-#endif
4697-#if (LLL_SHARED-LLL_PRIVATE) > 255
4698- xorl %ecx, %ecx
4699-#endif
4700- cmpl $-1, dep_mutex(%ebx)
4701- setne %cl
4702- subl $1, %ecx
4703- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4704-#if LLL_PRIVATE != 0
4705- addl $LLL_PRIVATE, %ecx
4706-#endif
4707- call __lll_lock_wait
4708-
4709-1: movl broadcast_seq(%ebx), %eax
4710- cmpl 12(%esp), %eax
4711- jne 3f
4712-
4713- /* We increment the wakeup_seq counter only if it is lower than
4714- total_seq. If this is not the case the thread was woken and
4715- then canceled. In this case we ignore the signal. */
4716- movl total_seq(%ebx), %eax
4717- movl total_seq+4(%ebx), %edi
4718- cmpl wakeup_seq+4(%ebx), %edi
4719- jb 6f
4720- ja 7f
4721- cmpl wakeup_seq(%ebx), %eax
4722- jbe 7f
4723-
4724-6: addl $1, wakeup_seq(%ebx)
4725- adcl $0, wakeup_seq+4(%ebx)
4726- addl $1, cond_futex(%ebx)
4727-
4728-7: addl $1, woken_seq(%ebx)
4729- adcl $0, woken_seq+4(%ebx)
4730-
4731-3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
4732-
4733- /* Wake up a thread which wants to destroy the condvar object. */
4734- xorl %edi, %edi
4735- movl total_seq(%ebx), %eax
4736- andl total_seq+4(%ebx), %eax
4737- cmpl $0xffffffff, %eax
4738- jne 4f
4739- movl cond_nwaiters(%ebx), %eax
4740- andl $~((1 << nwaiters_shift) - 1), %eax
4741- jne 4f
4742-
4743- addl $cond_nwaiters, %ebx
4744- movl $SYS_futex, %eax
4745-#if FUTEX_PRIVATE_FLAG > 255
4746- xorl %ecx, %ecx
4747-#endif
4748- cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
4749- sete %cl
4750- subl $1, %ecx
4751-#ifdef __ASSUME_PRIVATE_FUTEX
4752- andl $FUTEX_PRIVATE_FLAG, %ecx
4753-#else
4754- andl %gs:PRIVATE_FUTEX, %ecx
4755-#endif
4756- addl $FUTEX_WAKE, %ecx
4757- movl $1, %edx
4758- ENTER_KERNEL
4759- subl $cond_nwaiters, %ebx
4760- movl $1, %edi
4761-
4762-4: LOCK
4763-#if cond_lock == 0
4764- subl $1, (%ebx)
4765-#else
4766- subl $1, cond_lock(%ebx)
4767-#endif
4768- je 2f
4769-
4770-#if cond_lock == 0
4771- movl %ebx, %eax
4772-#else
4773- leal cond_lock(%ebx), %eax
4774-#endif
4775-#if (LLL_SHARED-LLL_PRIVATE) > 255
4776- xorl %ecx, %ecx
4777-#endif
4778- cmpl $-1, dep_mutex(%ebx)
4779- setne %cl
4780- subl $1, %ecx
4781- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
4782-#if LLL_PRIVATE != 0
4783- addl $LLL_PRIVATE, %ecx
4784-#endif
4785- call __lll_unlock_wake
4786-
4787- /* Wake up all waiters to make sure no signal gets lost. */
4788-2: testl %edi, %edi
4789- jnz 5f
4790- addl $cond_futex, %ebx
4791-#if FUTEX_PRIVATE_FLAG > 255
4792- xorl %ecx, %ecx
4793-#endif
4794- cmpl $-1, dep_mutex-cond_futex(%ebx)
4795- sete %cl
4796- subl $1, %ecx
4797-#ifdef __ASSUME_PRIVATE_FUTEX
4798- andl $FUTEX_PRIVATE_FLAG, %ecx
4799-#else
4800- andl %gs:PRIVATE_FUTEX, %ecx
4801-#endif
4802- addl $FUTEX_WAKE, %ecx
4803- movl $SYS_futex, %eax
4804- movl $0x7fffffff, %edx
4805- ENTER_KERNEL
4806-
4807- /* Lock the mutex only if we don't own it already. This only happens
4808- in case of PI mutexes, if we got cancelled after a successful
4809- return of the futex syscall and before disabling async
4810- cancellation. */
4811-5: movl 24+FRAME_SIZE(%esp), %eax
4812- movl MUTEX_KIND(%eax), %ebx
4813- andl $(ROBUST_BIT|PI_BIT), %ebx
4814- cmpl $PI_BIT, %ebx
4815- jne 8f
4816-
4817- movl (%eax), %ebx
4818- andl $TID_MASK, %ebx
4819- cmpl %ebx, %gs:TID
4820- jne 8f
4821- /* We managed to get the lock. Fix it up before returning. */
4822- call __pthread_mutex_cond_lock_adjust
4823- jmp 9f
4824-
4825-8: call __pthread_mutex_cond_lock
4826-
4827-9: movl %esi, (%esp)
4828-.LcallUR:
4829- call _Unwind_Resume
4830- hlt
4831-.LENDCODE:
4832- cfi_endproc
4833- .size __condvar_tw_cleanup, .-__condvar_tw_cleanup
4834-
4835-
4836- .section .gcc_except_table,"a",@progbits
4837-.LexceptSTART:
4838- .byte DW_EH_PE_omit # @LPStart format (omit)
4839- .byte DW_EH_PE_omit # @TType format (omit)
4840- .byte DW_EH_PE_sdata4 # call-site format
4841- # DW_EH_PE_sdata4
4842- .uleb128 .Lcstend-.Lcstbegin
4843-.Lcstbegin:
4844- .long .LcleanupSTART-.LSTARTCODE
4845- .long .Ladd_cond_futex_pi-.LcleanupSTART
4846- .long __condvar_tw_cleanup-.LSTARTCODE
4847- .uleb128 0
4848- .long .Ladd_cond_futex_pi-.LSTARTCODE
4849- .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi
4850- .long __condvar_tw_cleanup2-.LSTARTCODE
4851- .uleb128 0
4852- .long .Lsub_cond_futex_pi-.LSTARTCODE
4853- .long .Ladd_cond_futex-.Lsub_cond_futex_pi
4854- .long __condvar_tw_cleanup-.LSTARTCODE
4855- .uleb128 0
4856- .long .Ladd_cond_futex-.LSTARTCODE
4857- .long .Lsub_cond_futex-.Ladd_cond_futex
4858- .long __condvar_tw_cleanup2-.LSTARTCODE
4859- .uleb128 0
4860- .long .Lsub_cond_futex-.LSTARTCODE
4861- .long .LcleanupEND-.Lsub_cond_futex
4862- .long __condvar_tw_cleanup-.LSTARTCODE
4863- .uleb128 0
4864-#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
4865- .long .LcleanupSTART2-.LSTARTCODE
4866- .long .Ladd_cond_futex2-.LcleanupSTART2
4867- .long __condvar_tw_cleanup-.LSTARTCODE
4868- .uleb128 0
4869- .long .Ladd_cond_futex2-.LSTARTCODE
4870- .long .Lsub_cond_futex2-.Ladd_cond_futex2
4871- .long __condvar_tw_cleanup2-.LSTARTCODE
4872- .uleb128 0
4873- .long .Lsub_cond_futex2-.LSTARTCODE
4874- .long .LcleanupEND2-.Lsub_cond_futex2
4875- .long __condvar_tw_cleanup-.LSTARTCODE
4876- .uleb128 0
4877-#endif
4878- .long .LcallUR-.LSTARTCODE
4879- .long .LENDCODE-.LcallUR
4880- .long 0
4881- .uleb128 0
4882-.Lcstend:
4883-
4884-
4885-#ifdef SHARED
4886- .hidden DW.ref.__gcc_personality_v0
4887- .weak DW.ref.__gcc_personality_v0
4888- .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
4889- .align 4
4890- .type DW.ref.__gcc_personality_v0, @object
4891- .size DW.ref.__gcc_personality_v0, 4
4892-DW.ref.__gcc_personality_v0:
4893- .long __gcc_personality_v0
4894-#endif
4895diff --git a/sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S b/sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S
4896deleted file mode 100644
4897index 5016718..0000000
4898--- a/sysdeps/unix/sysv/linux/i386/pthread_cond_wait.S
4899+++ /dev/null
4900@@ -1,642 +0,0 @@
4901-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
4902- This file is part of the GNU C Library.
4903- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4904-
4905- The GNU C Library is free software; you can redistribute it and/or
4906- modify it under the terms of the GNU Lesser General Public
4907- License as published by the Free Software Foundation; either
4908- version 2.1 of the License, or (at your option) any later version.
4909-
4910- The GNU C Library is distributed in the hope that it will be useful,
4911- but WITHOUT ANY WARRANTY; without even the implied warranty of
4912- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
4913- Lesser General Public License for more details.
4914-
4915- You should have received a copy of the GNU Lesser General Public
4916- License along with the GNU C Library; if not, see
4917- <http://www.gnu.org/licenses/>. */
4918-
4919-#include <sysdep.h>
4920-#include <shlib-compat.h>
4921-#include <lowlevellock.h>
4922-#include <lowlevelcond.h>
4923-#include <tcb-offsets.h>
4924-#include <pthread-errnos.h>
4925-#include <pthread-pi-defines.h>
4926-#include <kernel-features.h>
4927-#include <stap-probe.h>
4928-
4929-
4930- .text
4931-
4932-/* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */
4933- .globl __pthread_cond_wait
4934- .type __pthread_cond_wait, @function
4935- .align 16
4936-__pthread_cond_wait:
4937-.LSTARTCODE:
4938- cfi_startproc
4939-#ifdef SHARED
4940- cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
4941- DW.ref.__gcc_personality_v0)
4942- cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
4943-#else
4944- cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
4945- cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
4946-#endif
4947-
4948- pushl %ebp
4949- cfi_adjust_cfa_offset(4)
4950- cfi_rel_offset(%ebp, 0)
4951- pushl %edi
4952- cfi_adjust_cfa_offset(4)
4953- cfi_rel_offset(%edi, 0)
4954- pushl %esi
4955- cfi_adjust_cfa_offset(4)
4956- cfi_rel_offset(%esi, 0)
4957- pushl %ebx
4958- cfi_adjust_cfa_offset(4)
4959- cfi_rel_offset(%ebx, 0)
4960-
4961- xorl %esi, %esi
4962- movl 20(%esp), %ebx
4963-
4964- LIBC_PROBE (cond_wait, 2, 24(%esp), %ebx)
4965-
4966- /* Get internal lock. */
4967- movl $1, %edx
4968- xorl %eax, %eax
4969- LOCK
4970-#if cond_lock == 0
4971- cmpxchgl %edx, (%ebx)
4972-#else
4973- cmpxchgl %edx, cond_lock(%ebx)
4974-#endif
4975- jnz 1f
4976-
4977- /* Store the reference to the mutex. If there is already a
4978- different value in there this is a bad user bug. */
4979-2: cmpl $-1, dep_mutex(%ebx)
4980- movl 24(%esp), %eax
4981- je 15f
4982- movl %eax, dep_mutex(%ebx)
4983-
4984- /* Unlock the mutex. */
4985-15: xorl %edx, %edx
4986- call __pthread_mutex_unlock_usercnt
4987-
4988- testl %eax, %eax
4989- jne 12f
4990-
4991- addl $1, total_seq(%ebx)
4992- adcl $0, total_seq+4(%ebx)
4993- addl $1, cond_futex(%ebx)
4994- addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
4995-
4996-#define FRAME_SIZE 20
4997- subl $FRAME_SIZE, %esp
4998- cfi_adjust_cfa_offset(FRAME_SIZE)
4999- cfi_remember_state
5000-
5001- /* Get and store current wakeup_seq value. */
5002- movl wakeup_seq(%ebx), %edi
5003- movl wakeup_seq+4(%ebx), %edx
5004- movl broadcast_seq(%ebx), %eax
5005- movl %edi, 4(%esp)
5006- movl %edx, 8(%esp)
5007- movl %eax, 12(%esp)
5008-
5009- /* Reset the pi-requeued flag. */
5010-8: movl $0, 16(%esp)
5011- movl cond_futex(%ebx), %ebp
5012-
5013- /* Unlock. */
5014- LOCK
5015-#if cond_lock == 0
5016- subl $1, (%ebx)
5017-#else
5018- subl $1, cond_lock(%ebx)
5019-#endif
5020- jne 3f
5021-
5022-.LcleanupSTART:
5023-4: call __pthread_enable_asynccancel
5024- movl %eax, (%esp)
5025-
5026- xorl %ecx, %ecx
5027- cmpl $-1, dep_mutex(%ebx)
5028- sete %cl
5029- je 18f
5030-
5031- movl dep_mutex(%ebx), %edi
5032- /* Requeue to a non-robust PI mutex if the PI bit is set and
5033- the robust bit is not set. */
5034- movl MUTEX_KIND(%edi), %eax
5035- andl $(ROBUST_BIT|PI_BIT), %eax
5036- cmpl $PI_BIT, %eax
5037- jne 18f
5038-
5039- movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
5040- movl %ebp, %edx
5041- xorl %esi, %esi
5042- addl $cond_futex, %ebx
5043-.Ladd_cond_futex_pi:
5044- movl $SYS_futex, %eax
5045- ENTER_KERNEL
5046- subl $cond_futex, %ebx
5047-.Lsub_cond_futex_pi:
5048- /* Set the pi-requeued flag only if the kernel has returned 0. The
5049- kernel does not hold the mutex on error. */
5050- cmpl $0, %eax
5051- sete 16(%esp)
5052- je 19f
5053-
5054- /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
5055- successfully, it has already locked the mutex for us and the
5056- pi_flag (16(%esp)) is set to denote that fact. However, if another
5057- thread changed the futex value before we entered the wait, the
5058- syscall may return an EAGAIN and the mutex is not locked. We go
5059- ahead with a success anyway since later we look at the pi_flag to
5060- decide if we got the mutex or not. The sequence numbers then make
5061- sure that only one of the threads actually wake up. We retry using
5062- normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
5063- and PI futexes don't mix.
5064-
5065- Note that we don't check for EAGAIN specifically; we assume that the
5066- only other error the futex function could return is EAGAIN since
5067- anything else would mean an error in our function. It is too
5068- expensive to do that check for every call (which is quite common in
5069- case of a large number of threads), so it has been skipped. */
5070- cmpl $-ENOSYS, %eax
5071- jne 19f
5072- xorl %ecx, %ecx
5073-
5074-18: subl $1, %ecx
5075-#ifdef __ASSUME_PRIVATE_FUTEX
5076- andl $FUTEX_PRIVATE_FLAG, %ecx
5077-#else
5078- andl %gs:PRIVATE_FUTEX, %ecx
5079-#endif
5080-#if FUTEX_WAIT != 0
5081- addl $FUTEX_WAIT, %ecx
5082-#endif
5083- movl %ebp, %edx
5084- addl $cond_futex, %ebx
5085-.Ladd_cond_futex:
5086- movl $SYS_futex, %eax
5087- ENTER_KERNEL
5088- subl $cond_futex, %ebx
5089-.Lsub_cond_futex:
5090-
5091-19: movl (%esp), %eax
5092- call __pthread_disable_asynccancel
5093-.LcleanupEND:
5094-
5095- /* Lock. */
5096- movl $1, %edx
5097- xorl %eax, %eax
5098- LOCK
5099-#if cond_lock == 0
5100- cmpxchgl %edx, (%ebx)
5101-#else
5102- cmpxchgl %edx, cond_lock(%ebx)
5103-#endif
5104- jnz 5f
5105-
5106-6: movl broadcast_seq(%ebx), %eax
5107- cmpl 12(%esp), %eax
5108- jne 16f
5109-
5110- movl woken_seq(%ebx), %eax
5111- movl woken_seq+4(%ebx), %ecx
5112-
5113- movl wakeup_seq(%ebx), %edi
5114- movl wakeup_seq+4(%ebx), %edx
5115-
5116- cmpl 8(%esp), %edx
5117- jne 7f
5118- cmpl 4(%esp), %edi
5119- je 22f
5120-
5121-7: cmpl %ecx, %edx
5122- jne 9f
5123- cmp %eax, %edi
5124- je 22f
5125-
5126-9: addl $1, woken_seq(%ebx)
5127- adcl $0, woken_seq+4(%ebx)
5128-
5129- /* Unlock */
5130-16: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
5131-
5132- /* Wake up a thread which wants to destroy the condvar object. */
5133- movl total_seq(%ebx), %eax
5134- andl total_seq+4(%ebx), %eax
5135- cmpl $0xffffffff, %eax
5136- jne 17f
5137- movl cond_nwaiters(%ebx), %eax
5138- andl $~((1 << nwaiters_shift) - 1), %eax
5139- jne 17f
5140-
5141- addl $cond_nwaiters, %ebx
5142- movl $SYS_futex, %eax
5143-#if FUTEX_PRIVATE_FLAG > 255
5144- xorl %ecx, %ecx
5145-#endif
5146- cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
5147- sete %cl
5148- subl $1, %ecx
5149-#ifdef __ASSUME_PRIVATE_FUTEX
5150- andl $FUTEX_PRIVATE_FLAG, %ecx
5151-#else
5152- andl %gs:PRIVATE_FUTEX, %ecx
5153-#endif
5154- addl $FUTEX_WAKE, %ecx
5155- movl $1, %edx
5156- ENTER_KERNEL
5157- subl $cond_nwaiters, %ebx
5158-
5159-17: LOCK
5160-#if cond_lock == 0
5161- subl $1, (%ebx)
5162-#else
5163- subl $1, cond_lock(%ebx)
5164-#endif
5165- jne 10f
5166-
5167- /* With requeue_pi, the mutex lock is held in the kernel. */
5168-11: movl 24+FRAME_SIZE(%esp), %eax
5169- movl 16(%esp), %ecx
5170- testl %ecx, %ecx
5171- jnz 21f
5172-
5173- call __pthread_mutex_cond_lock
5174-20: addl $FRAME_SIZE, %esp
5175- cfi_adjust_cfa_offset(-FRAME_SIZE);
5176-
5177-14: popl %ebx
5178- cfi_adjust_cfa_offset(-4)
5179- cfi_restore(%ebx)
5180- popl %esi
5181- cfi_adjust_cfa_offset(-4)
5182- cfi_restore(%esi)
5183- popl %edi
5184- cfi_adjust_cfa_offset(-4)
5185- cfi_restore(%edi)
5186- popl %ebp
5187- cfi_adjust_cfa_offset(-4)
5188- cfi_restore(%ebp)
5189-
5190- /* We return the result of the mutex_lock operation. */
5191- ret
5192-
5193- cfi_restore_state
5194-
5195-21: call __pthread_mutex_cond_lock_adjust
5196- xorl %eax, %eax
5197- jmp 20b
5198-
5199- cfi_adjust_cfa_offset(-FRAME_SIZE);
5200-
5201- /* We need to go back to futex_wait. If we're using requeue_pi, then
5202- release the mutex we had acquired and go back. */
5203-22: movl 16(%esp), %edx
5204- test %edx, %edx
5205- jz 8b
5206-
5207- /* Adjust the mutex values first and then unlock it. The unlock
5208- should always succeed or else the kernel did not lock the mutex
5209- correctly. */
5210- movl dep_mutex(%ebx), %eax
5211- call __pthread_mutex_cond_lock_adjust
5212- movl dep_mutex(%ebx), %eax
5213- xorl %edx, %edx
5214- call __pthread_mutex_unlock_usercnt
5215- jmp 8b
5216-
5217- /* Initial locking failed. */
5218-1:
5219-#if cond_lock == 0
5220- movl %ebx, %edx
5221-#else
5222- leal cond_lock(%ebx), %edx
5223-#endif
5224-#if (LLL_SHARED-LLL_PRIVATE) > 255
5225- xorl %ecx, %ecx
5226-#endif
5227- cmpl $-1, dep_mutex(%ebx)
5228- setne %cl
5229- subl $1, %ecx
5230- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5231-#if LLL_PRIVATE != 0
5232- addl $LLL_PRIVATE, %ecx
5233-#endif
5234- call __lll_lock_wait
5235- jmp 2b
5236-
5237- /* The initial unlocking of the mutex failed. */
5238-12:
5239- LOCK
5240-#if cond_lock == 0
5241- subl $1, (%ebx)
5242-#else
5243- subl $1, cond_lock(%ebx)
5244-#endif
5245- jne 14b
5246-
5247- movl %eax, %esi
5248-#if cond_lock == 0
5249- movl %ebx, %eax
5250-#else
5251- leal cond_lock(%ebx), %eax
5252-#endif
5253-#if (LLL_SHARED-LLL_PRIVATE) > 255
5254- xorl %ecx, %ecx
5255-#endif
5256- cmpl $-1, dep_mutex(%ebx)
5257- setne %cl
5258- subl $1, %ecx
5259- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5260-#if LLL_PRIVATE != 0
5261- addl $LLL_PRIVATE, %ecx
5262-#endif
5263- call __lll_unlock_wake
5264-
5265- movl %esi, %eax
5266- jmp 14b
5267-
5268- cfi_adjust_cfa_offset(FRAME_SIZE)
5269-
5270- /* Unlock in loop requires wakeup. */
5271-3:
5272-#if cond_lock == 0
5273- movl %ebx, %eax
5274-#else
5275- leal cond_lock(%ebx), %eax
5276-#endif
5277-#if (LLL_SHARED-LLL_PRIVATE) > 255
5278- xorl %ecx, %ecx
5279-#endif
5280- cmpl $-1, dep_mutex(%ebx)
5281- setne %cl
5282- subl $1, %ecx
5283- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5284-#if LLL_PRIVATE != 0
5285- addl $LLL_PRIVATE, %ecx
5286-#endif
5287- call __lll_unlock_wake
5288- jmp 4b
5289-
5290- /* Locking in loop failed. */
5291-5:
5292-#if cond_lock == 0
5293- movl %ebx, %edx
5294-#else
5295- leal cond_lock(%ebx), %edx
5296-#endif
5297-#if (LLL_SHARED-LLL_PRIVATE) > 255
5298- xorl %ecx, %ecx
5299-#endif
5300- cmpl $-1, dep_mutex(%ebx)
5301- setne %cl
5302- subl $1, %ecx
5303- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5304-#if LLL_PRIVATE != 0
5305- addl $LLL_PRIVATE, %ecx
5306-#endif
5307- call __lll_lock_wait
5308- jmp 6b
5309-
5310- /* Unlock after loop requires wakeup. */
5311-10:
5312-#if cond_lock == 0
5313- movl %ebx, %eax
5314-#else
5315- leal cond_lock(%ebx), %eax
5316-#endif
5317-#if (LLL_SHARED-LLL_PRIVATE) > 255
5318- xorl %ecx, %ecx
5319-#endif
5320- cmpl $-1, dep_mutex(%ebx)
5321- setne %cl
5322- subl $1, %ecx
5323- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5324-#if LLL_PRIVATE != 0
5325- addl $LLL_PRIVATE, %ecx
5326-#endif
5327- call __lll_unlock_wake
5328- jmp 11b
5329-
5330- .size __pthread_cond_wait, .-__pthread_cond_wait
5331-versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
5332- GLIBC_2_3_2)
5333-
5334-
5335- .type __condvar_w_cleanup2, @function
5336-__condvar_w_cleanup2:
5337- subl $cond_futex, %ebx
5338- .size __condvar_w_cleanup2, .-__condvar_w_cleanup2
5339-.LSbl4:
5340- .type __condvar_w_cleanup, @function
5341-__condvar_w_cleanup:
5342- movl %eax, %esi
5343-
5344- /* Get internal lock. */
5345- movl $1, %edx
5346- xorl %eax, %eax
5347- LOCK
5348-#if cond_lock == 0
5349- cmpxchgl %edx, (%ebx)
5350-#else
5351- cmpxchgl %edx, cond_lock(%ebx)
5352-#endif
5353- jz 1f
5354-
5355-#if cond_lock == 0
5356- movl %ebx, %edx
5357-#else
5358- leal cond_lock(%ebx), %edx
5359-#endif
5360-#if (LLL_SHARED-LLL_PRIVATE) > 255
5361- xorl %ecx, %ecx
5362-#endif
5363- cmpl $-1, dep_mutex(%ebx)
5364- setne %cl
5365- subl $1, %ecx
5366- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5367-#if LLL_PRIVATE != 0
5368- addl $LLL_PRIVATE, %ecx
5369-#endif
5370- call __lll_lock_wait
5371-
5372-1: movl broadcast_seq(%ebx), %eax
5373- cmpl 12(%esp), %eax
5374- jne 3f
5375-
5376- /* We increment the wakeup_seq counter only if it is lower than
5377- total_seq. If this is not the case the thread was woken and
5378- then canceled. In this case we ignore the signal. */
5379- movl total_seq(%ebx), %eax
5380- movl total_seq+4(%ebx), %edi
5381- cmpl wakeup_seq+4(%ebx), %edi
5382- jb 6f
5383- ja 7f
5384- cmpl wakeup_seq(%ebx), %eax
5385- jbe 7f
5386-
5387-6: addl $1, wakeup_seq(%ebx)
5388- adcl $0, wakeup_seq+4(%ebx)
5389- addl $1, cond_futex(%ebx)
5390-
5391-7: addl $1, woken_seq(%ebx)
5392- adcl $0, woken_seq+4(%ebx)
5393-
5394-3: subl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
5395-
5396- /* Wake up a thread which wants to destroy the condvar object. */
5397- xorl %edi, %edi
5398- movl total_seq(%ebx), %eax
5399- andl total_seq+4(%ebx), %eax
5400- cmpl $0xffffffff, %eax
5401- jne 4f
5402- movl cond_nwaiters(%ebx), %eax
5403- andl $~((1 << nwaiters_shift) - 1), %eax
5404- jne 4f
5405-
5406- addl $cond_nwaiters, %ebx
5407- movl $SYS_futex, %eax
5408-#if FUTEX_PRIVATE_FLAG > 255
5409- xorl %ecx, %ecx
5410-#endif
5411- cmpl $-1, dep_mutex-cond_nwaiters(%ebx)
5412- sete %cl
5413- subl $1, %ecx
5414-#ifdef __ASSUME_PRIVATE_FUTEX
5415- andl $FUTEX_PRIVATE_FLAG, %ecx
5416-#else
5417- andl %gs:PRIVATE_FUTEX, %ecx
5418-#endif
5419- addl $FUTEX_WAKE, %ecx
5420- movl $1, %edx
5421- ENTER_KERNEL
5422- subl $cond_nwaiters, %ebx
5423- movl $1, %edi
5424-
5425-4: LOCK
5426-#if cond_lock == 0
5427- subl $1, (%ebx)
5428-#else
5429- subl $1, cond_lock(%ebx)
5430-#endif
5431- je 2f
5432-
5433-#if cond_lock == 0
5434- movl %ebx, %eax
5435-#else
5436- leal cond_lock(%ebx), %eax
5437-#endif
5438-#if (LLL_SHARED-LLL_PRIVATE) > 255
5439- xorl %ecx, %ecx
5440-#endif
5441- cmpl $-1, dep_mutex(%ebx)
5442- setne %cl
5443- subl $1, %ecx
5444- andl $(LLL_SHARED-LLL_PRIVATE), %ecx
5445-#if LLL_PRIVATE != 0
5446- addl $LLL_PRIVATE, %ecx
5447-#endif
5448- call __lll_unlock_wake
5449-
5450- /* Wake up all waiters to make sure no signal gets lost. */
5451-2: testl %edi, %edi
5452- jnz 5f
5453- addl $cond_futex, %ebx
5454-#if FUTEX_PRIVATE_FLAG > 255
5455- xorl %ecx, %ecx
5456-#endif
5457- cmpl $-1, dep_mutex-cond_futex(%ebx)
5458- sete %cl
5459- subl $1, %ecx
5460-#ifdef __ASSUME_PRIVATE_FUTEX
5461- andl $FUTEX_PRIVATE_FLAG, %ecx
5462-#else
5463- andl %gs:PRIVATE_FUTEX, %ecx
5464-#endif
5465- addl $FUTEX_WAKE, %ecx
5466- movl $SYS_futex, %eax
5467- movl $0x7fffffff, %edx
5468- ENTER_KERNEL
5469-
5470- /* Lock the mutex only if we don't own it already. This only happens
5471- in case of PI mutexes, if we got cancelled after a successful
5472- return of the futex syscall and before disabling async
5473- cancellation. */
5474-5: movl 24+FRAME_SIZE(%esp), %eax
5475- movl MUTEX_KIND(%eax), %ebx
5476- andl $(ROBUST_BIT|PI_BIT), %ebx
5477- cmpl $PI_BIT, %ebx
5478- jne 8f
5479-
5480- movl (%eax), %ebx
5481- andl $TID_MASK, %ebx
5482- cmpl %ebx, %gs:TID
5483- jne 8f
5484- /* We managed to get the lock. Fix it up before returning. */
5485- call __pthread_mutex_cond_lock_adjust
5486- jmp 9f
5487-
5488-8: call __pthread_mutex_cond_lock
5489-
5490-9: movl %esi, (%esp)
5491-.LcallUR:
5492- call _Unwind_Resume
5493- hlt
5494-.LENDCODE:
5495- cfi_endproc
5496- .size __condvar_w_cleanup, .-__condvar_w_cleanup
5497-
5498-
5499- .section .gcc_except_table,"a",@progbits
5500-.LexceptSTART:
5501- .byte DW_EH_PE_omit # @LPStart format (omit)
5502- .byte DW_EH_PE_omit # @TType format (omit)
5503- .byte DW_EH_PE_sdata4 # call-site format
5504- # DW_EH_PE_sdata4
5505- .uleb128 .Lcstend-.Lcstbegin
5506-.Lcstbegin:
5507- .long .LcleanupSTART-.LSTARTCODE
5508- .long .Ladd_cond_futex_pi-.LcleanupSTART
5509- .long __condvar_w_cleanup-.LSTARTCODE
5510- .uleb128 0
5511- .long .Ladd_cond_futex_pi-.LSTARTCODE
5512- .long .Lsub_cond_futex_pi-.Ladd_cond_futex_pi
5513- .long __condvar_w_cleanup2-.LSTARTCODE
5514- .uleb128 0
5515- .long .Lsub_cond_futex_pi-.LSTARTCODE
5516- .long .Ladd_cond_futex-.Lsub_cond_futex_pi
5517- .long __condvar_w_cleanup-.LSTARTCODE
5518- .uleb128 0
5519- .long .Ladd_cond_futex-.LSTARTCODE
5520- .long .Lsub_cond_futex-.Ladd_cond_futex
5521- .long __condvar_w_cleanup2-.LSTARTCODE
5522- .uleb128 0
5523- .long .Lsub_cond_futex-.LSTARTCODE
5524- .long .LcleanupEND-.Lsub_cond_futex
5525- .long __condvar_w_cleanup-.LSTARTCODE
5526- .uleb128 0
5527- .long .LcallUR-.LSTARTCODE
5528- .long .LENDCODE-.LcallUR
5529- .long 0
5530- .uleb128 0
5531-.Lcstend:
5532-
5533-#ifdef SHARED
5534- .hidden DW.ref.__gcc_personality_v0
5535- .weak DW.ref.__gcc_personality_v0
5536- .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
5537- .align 4
5538- .type DW.ref.__gcc_personality_v0, @object
5539- .size DW.ref.__gcc_personality_v0, 4
5540-DW.ref.__gcc_personality_v0:
5541- .long __gcc_personality_v0
5542-#endif
5543diff --git a/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h b/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
5544index 345e79a..371bc3c 100644
5545--- a/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
5546+++ b/sysdeps/unix/sysv/linux/powerpc/bits/pthreadtypes.h
5547@@ -123,19 +123,32 @@ typedef union
5548
5549
5550 /* Data structure for conditional variable handling. The structure of
5551- the attribute type is deliberately not exposed. */
5552+ the attribute type is not exposed on purpose. */
5553 typedef union
5554 {
5555 struct
5556 {
5557- int __lock;
5558- unsigned int __futex;
5559- __extension__ unsigned long long int __total_seq;
5560- __extension__ unsigned long long int __wakeup_seq;
5561- __extension__ unsigned long long int __woken_seq;
5562- void *__mutex;
5563- unsigned int __nwaiters;
5564- unsigned int __broadcast_seq;
5565+ __extension__ union
5566+ {
5567+ __extension__ unsigned long long int __wseq;
5568+ struct {
5569+ unsigned int __low;
5570+ unsigned int __high;
5571+ } __wseq32;
5572+ };
5573+ __extension__ union
5574+ {
5575+ __extension__ unsigned long long int __g1_start;
5576+ struct {
5577+ unsigned int __low;
5578+ unsigned int __high;
5579+ } __g1_start32;
5580+ };
5581+ unsigned int __g_refs[2];
5582+ unsigned int __g_size[2];
5583+ unsigned int __g1_orig_size;
5584+ unsigned int __wrefs;
5585+ unsigned int __g_signals[2];
5586 } __data;
5587 char __size[__SIZEOF_PTHREAD_COND_T];
5588 __extension__ long long int __align;
5589diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
5590deleted file mode 100644
5591index de455dd..0000000
5592--- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S
5593+++ /dev/null
5594@@ -1,177 +0,0 @@
5595-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
5596- This file is part of the GNU C Library.
5597- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5598-
5599- The GNU C Library is free software; you can redistribute it and/or
5600- modify it under the terms of the GNU Lesser General Public
5601- License as published by the Free Software Foundation; either
5602- version 2.1 of the License, or (at your option) any later version.
5603-
5604- The GNU C Library is distributed in the hope that it will be useful,
5605- but WITHOUT ANY WARRANTY; without even the implied warranty of
5606- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5607- Lesser General Public License for more details.
5608-
5609- You should have received a copy of the GNU Lesser General Public
5610- License along with the GNU C Library; if not, see
5611- <http://www.gnu.org/licenses/>. */
5612-
5613-#include <sysdep.h>
5614-#include <shlib-compat.h>
5615-#include <lowlevellock.h>
5616-#include <lowlevelcond.h>
5617-#include <kernel-features.h>
5618-#include <pthread-pi-defines.h>
5619-#include <pthread-errnos.h>
5620-#include <stap-probe.h>
5621-
5622- .text
5623-
5624- /* int pthread_cond_broadcast (pthread_cond_t *cond) */
5625-ENTRY(__pthread_cond_broadcast)
5626-
5627- LIBC_PROBE (cond_broadcast, 1, %rdi)
5628-
5629- /* Get internal lock. */
5630- movl $1, %esi
5631- xorl %eax, %eax
5632- LOCK
5633-#if cond_lock == 0
5634- cmpxchgl %esi, (%rdi)
5635-#else
5636- cmpxchgl %esi, cond_lock(%rdi)
5637-#endif
5638- jnz 1f
5639-
5640-2: addq $cond_futex, %rdi
5641- movq total_seq-cond_futex(%rdi), %r9
5642- cmpq wakeup_seq-cond_futex(%rdi), %r9
5643- jna 4f
5644-
5645- /* Cause all currently waiting threads to recognize they are
5646- woken up. */
5647- movq %r9, wakeup_seq-cond_futex(%rdi)
5648- movq %r9, woken_seq-cond_futex(%rdi)
5649- addq %r9, %r9
5650- movl %r9d, (%rdi)
5651- incl broadcast_seq-cond_futex(%rdi)
5652-
5653- /* Get the address of the mutex used. */
5654- mov dep_mutex-cond_futex(%rdi), %R8_LP
5655-
5656- /* Unlock. */
5657- LOCK
5658- decl cond_lock-cond_futex(%rdi)
5659- jne 7f
5660-
5661-8: cmp $-1, %R8_LP
5662- je 9f
5663-
5664- /* Do not use requeue for pshared condvars. */
5665- testl $PS_BIT, MUTEX_KIND(%r8)
5666- jne 9f
5667-
5668- /* Requeue to a PI mutex if the PI bit is set. */
5669- movl MUTEX_KIND(%r8), %eax
5670- andl $(ROBUST_BIT|PI_BIT), %eax
5671- cmpl $PI_BIT, %eax
5672- je 81f
5673-
5674- /* Wake up all threads. */
5675-#ifdef __ASSUME_PRIVATE_FUTEX
5676- movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi
5677-#else
5678- movl %fs:PRIVATE_FUTEX, %esi
5679- orl $FUTEX_CMP_REQUEUE, %esi
5680-#endif
5681- movl $SYS_futex, %eax
5682- movl $1, %edx
5683- movl $0x7fffffff, %r10d
5684- syscall
5685-
5686- /* For any kind of error, which mainly is EAGAIN, we try again
5687- with WAKE. The general test also covers running on old
5688- kernels. */
5689- cmpq $-4095, %rax
5690- jae 9f
5691-
5692-10: xorl %eax, %eax
5693- retq
5694-
5695- /* Wake up all threads. */
5696-81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
5697- movl $SYS_futex, %eax
5698- movl $1, %edx
5699- movl $0x7fffffff, %r10d
5700- syscall
5701-
5702- /* For any kind of error, which mainly is EAGAIN, we try again
5703- with WAKE. The general test also covers running on old
5704- kernels. */
5705- cmpq $-4095, %rax
5706- jb 10b
5707- jmp 9f
5708-
5709- .align 16
5710- /* Unlock. */
5711-4: LOCK
5712- decl cond_lock-cond_futex(%rdi)
5713- jne 5f
5714-
5715-6: xorl %eax, %eax
5716- retq
5717-
5718- /* Initial locking failed. */
5719-1:
5720-#if cond_lock != 0
5721- addq $cond_lock, %rdi
5722-#endif
5723- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
5724- movl $LLL_PRIVATE, %eax
5725- movl $LLL_SHARED, %esi
5726- cmovne %eax, %esi
5727- callq __lll_lock_wait
5728-#if cond_lock != 0
5729- subq $cond_lock, %rdi
5730-#endif
5731- jmp 2b
5732-
5733- /* Unlock in loop requires wakeup. */
5734-5: addq $cond_lock-cond_futex, %rdi
5735- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
5736- movl $LLL_PRIVATE, %eax
5737- movl $LLL_SHARED, %esi
5738- cmovne %eax, %esi
5739- callq __lll_unlock_wake
5740- jmp 6b
5741-
5742- /* Unlock in loop requires wakeup. */
5743-7: addq $cond_lock-cond_futex, %rdi
5744- cmp $-1, %R8_LP
5745- movl $LLL_PRIVATE, %eax
5746- movl $LLL_SHARED, %esi
5747- cmovne %eax, %esi
5748- callq __lll_unlock_wake
5749- subq $cond_lock-cond_futex, %rdi
5750- jmp 8b
5751-
5752-9: /* The futex requeue functionality is not available. */
5753- cmp $-1, %R8_LP
5754- movl $0x7fffffff, %edx
5755-#ifdef __ASSUME_PRIVATE_FUTEX
5756- movl $FUTEX_WAKE, %eax
5757- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
5758- cmove %eax, %esi
5759-#else
5760- movl $0, %eax
5761- movl %fs:PRIVATE_FUTEX, %esi
5762- cmove %eax, %esi
5763- orl $FUTEX_WAKE, %esi
5764-#endif
5765- movl $SYS_futex, %eax
5766- syscall
5767- jmp 10b
5768-END(__pthread_cond_broadcast)
5769-
5770-versioned_symbol (libpthread, __pthread_cond_broadcast, pthread_cond_broadcast,
5771- GLIBC_2_3_2)
5772diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
5773deleted file mode 100644
5774index da14bc3..0000000
5775--- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S
5776+++ /dev/null
5777@@ -1,161 +0,0 @@
5778-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
5779- This file is part of the GNU C Library.
5780- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5781-
5782- The GNU C Library is free software; you can redistribute it and/or
5783- modify it under the terms of the GNU Lesser General Public
5784- License as published by the Free Software Foundation; either
5785- version 2.1 of the License, or (at your option) any later version.
5786-
5787- The GNU C Library is distributed in the hope that it will be useful,
5788- but WITHOUT ANY WARRANTY; without even the implied warranty of
5789- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5790- Lesser General Public License for more details.
5791-
5792- You should have received a copy of the GNU Lesser General Public
5793- License along with the GNU C Library; if not, see
5794- <http://www.gnu.org/licenses/>. */
5795-
5796-#include <sysdep.h>
5797-#include <shlib-compat.h>
5798-#include <lowlevellock.h>
5799-#include <lowlevelcond.h>
5800-#include <pthread-pi-defines.h>
5801-#include <kernel-features.h>
5802-#include <pthread-errnos.h>
5803-#include <stap-probe.h>
5804-
5805-
5806- .text
5807-
5808-ENTRY(__pthread_cond_signal)
5809-
5810- LIBC_PROBE (cond_signal, 1, %rdi)
5811-
5812- /* Get internal lock. */
5813- movq %rdi, %r8
5814- movl $1, %esi
5815- xorl %eax, %eax
5816- LOCK
5817-#if cond_lock == 0
5818- cmpxchgl %esi, (%rdi)
5819-#else
5820- cmpxchgl %esi, cond_lock(%rdi)
5821-#endif
5822- jnz 1f
5823-
5824-2: addq $cond_futex, %rdi
5825- movq total_seq(%r8), %rcx
5826- cmpq wakeup_seq(%r8), %rcx
5827- jbe 4f
5828-
5829- /* Bump the wakeup number. */
5830- addq $1, wakeup_seq(%r8)
5831- addl $1, (%rdi)
5832-
5833- /* Wake up one thread. */
5834- LP_OP(cmp) $-1, dep_mutex(%r8)
5835- movl $FUTEX_WAKE_OP, %esi
5836- movl $1, %edx
5837- movl $SYS_futex, %eax
5838- je 8f
5839-
5840- /* Get the address of the mutex used. */
5841- mov dep_mutex(%r8), %RCX_LP
5842- movl MUTEX_KIND(%rcx), %r11d
5843- andl $(ROBUST_BIT|PI_BIT), %r11d
5844- cmpl $PI_BIT, %r11d
5845- je 9f
5846-
5847-#ifdef __ASSUME_PRIVATE_FUTEX
5848- movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi
5849-#else
5850- orl %fs:PRIVATE_FUTEX, %esi
5851-#endif
5852-
5853-8: movl $1, %r10d
5854-#if cond_lock != 0
5855- addq $cond_lock, %r8
5856-#endif
5857- movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %r9d
5858- syscall
5859-#if cond_lock != 0
5860- subq $cond_lock, %r8
5861-#endif
5862- /* For any kind of error, we try again with WAKE.
5863- The general test also covers running on old kernels. */
5864- cmpq $-4095, %rax
5865- jae 7f
5866-
5867- xorl %eax, %eax
5868- retq
5869-
5870- /* Wake up one thread and requeue none in the PI Mutex case. */
5871-9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
5872- movq %rcx, %r8
5873- xorq %r10, %r10
5874- movl (%rdi), %r9d // XXX Can this be right?
5875- syscall
5876-
5877- leaq -cond_futex(%rdi), %r8
5878-
5879- /* For any kind of error, we try again with WAKE.
5880- The general test also covers running on old kernels. */
5881- cmpq $-4095, %rax
5882- jb 4f
5883-
5884-7:
5885-#ifdef __ASSUME_PRIVATE_FUTEX
5886- andl $FUTEX_PRIVATE_FLAG, %esi
5887-#else
5888- andl %fs:PRIVATE_FUTEX, %esi
5889-#endif
5890- orl $FUTEX_WAKE, %esi
5891- movl $SYS_futex, %eax
5892- /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall.
5893- movl $1, %edx */
5894- syscall
5895-
5896- /* Unlock. */
5897-4: LOCK
5898-#if cond_lock == 0
5899- decl (%r8)
5900-#else
5901- decl cond_lock(%r8)
5902-#endif
5903- jne 5f
5904-
5905-6: xorl %eax, %eax
5906- retq
5907-
5908- /* Initial locking failed. */
5909-1:
5910-#if cond_lock != 0
5911- addq $cond_lock, %rdi
5912-#endif
5913- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
5914- movl $LLL_PRIVATE, %eax
5915- movl $LLL_SHARED, %esi
5916- cmovne %eax, %esi
5917- callq __lll_lock_wait
5918-#if cond_lock != 0
5919- subq $cond_lock, %rdi
5920-#endif
5921- jmp 2b
5922-
5923- /* Unlock in loop requires wakeup. */
5924-5:
5925- movq %r8, %rdi
5926-#if cond_lock != 0
5927- addq $cond_lock, %rdi
5928-#endif
5929- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
5930- movl $LLL_PRIVATE, %eax
5931- movl $LLL_SHARED, %esi
5932- cmovne %eax, %esi
5933- callq __lll_unlock_wake
5934- jmp 6b
5935-END(__pthread_cond_signal)
5936-
5937-versioned_symbol (libpthread, __pthread_cond_signal, pthread_cond_signal,
5938- GLIBC_2_3_2)
5939diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
5940deleted file mode 100644
5941index 82ffa1a..0000000
5942--- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S
5943+++ /dev/null
5944@@ -1,623 +0,0 @@
5945-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
5946- This file is part of the GNU C Library.
5947- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5948-
5949- The GNU C Library is free software; you can redistribute it and/or
5950- modify it under the terms of the GNU Lesser General Public
5951- License as published by the Free Software Foundation; either
5952- version 2.1 of the License, or (at your option) any later version.
5953-
5954- The GNU C Library is distributed in the hope that it will be useful,
5955- but WITHOUT ANY WARRANTY; without even the implied warranty of
5956- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
5957- Lesser General Public License for more details.
5958-
5959- You should have received a copy of the GNU Lesser General Public
5960- License along with the GNU C Library; if not, see
5961- <http://www.gnu.org/licenses/>. */
5962-
5963-#include <sysdep.h>
5964-#include <shlib-compat.h>
5965-#include <lowlevellock.h>
5966-#include <lowlevelcond.h>
5967-#include <pthread-pi-defines.h>
5968-#include <pthread-errnos.h>
5969-#include <stap-probe.h>
5970-
5971-#include <kernel-features.h>
5972-
5973-
5974- .text
5975-
5976-
5977-/* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex,
5978- const struct timespec *abstime) */
5979- .globl __pthread_cond_timedwait
5980- .type __pthread_cond_timedwait, @function
5981- .align 16
5982-__pthread_cond_timedwait:
5983-.LSTARTCODE:
5984- cfi_startproc
5985-#ifdef SHARED
5986- cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
5987- DW.ref.__gcc_personality_v0)
5988- cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
5989-#else
5990- cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
5991- cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
5992-#endif
5993-
5994- pushq %r12
5995- cfi_adjust_cfa_offset(8)
5996- cfi_rel_offset(%r12, 0)
5997- pushq %r13
5998- cfi_adjust_cfa_offset(8)
5999- cfi_rel_offset(%r13, 0)
6000- pushq %r14
6001- cfi_adjust_cfa_offset(8)
6002- cfi_rel_offset(%r14, 0)
6003- pushq %r15
6004- cfi_adjust_cfa_offset(8)
6005- cfi_rel_offset(%r15, 0)
6006-#define FRAME_SIZE (32+8)
6007- subq $FRAME_SIZE, %rsp
6008- cfi_adjust_cfa_offset(FRAME_SIZE)
6009- cfi_remember_state
6010-
6011- LIBC_PROBE (cond_timedwait, 3, %rdi, %rsi, %rdx)
6012-
6013- cmpq $1000000000, 8(%rdx)
6014- movl $EINVAL, %eax
6015- jae 48f
6016-
6017- /* Stack frame:
6018-
6019- rsp + 48
6020- +--------------------------+
6021- rsp + 32 | timeout value |
6022- +--------------------------+
6023- rsp + 24 | old wake_seq value |
6024- +--------------------------+
6025- rsp + 16 | mutex pointer |
6026- +--------------------------+
6027- rsp + 8 | condvar pointer |
6028- +--------------------------+
6029- rsp + 4 | old broadcast_seq value |
6030- +--------------------------+
6031- rsp + 0 | old cancellation mode |
6032- +--------------------------+
6033- */
6034-
6035- LP_OP(cmp) $-1, dep_mutex(%rdi)
6036-
6037- /* Prepare structure passed to cancellation handler. */
6038- movq %rdi, 8(%rsp)
6039- movq %rsi, 16(%rsp)
6040- movq %rdx, %r13
6041-
6042- je 22f
6043- mov %RSI_LP, dep_mutex(%rdi)
6044-
6045-22:
6046- xorb %r15b, %r15b
6047-
6048- /* Get internal lock. */
6049- movl $1, %esi
6050- xorl %eax, %eax
6051- LOCK
6052-#if cond_lock == 0
6053- cmpxchgl %esi, (%rdi)
6054-#else
6055- cmpxchgl %esi, cond_lock(%rdi)
6056-#endif
6057- jnz 31f
6058-
6059- /* Unlock the mutex. */
6060-32: movq 16(%rsp), %rdi
6061- xorl %esi, %esi
6062- callq __pthread_mutex_unlock_usercnt
6063-
6064- testl %eax, %eax
6065- jne 46f
6066-
6067- movq 8(%rsp), %rdi
6068- incq total_seq(%rdi)
6069- incl cond_futex(%rdi)
6070- addl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
6071-
6072- /* Get and store current wakeup_seq value. */
6073- movq 8(%rsp), %rdi
6074- movq wakeup_seq(%rdi), %r9
6075- movl broadcast_seq(%rdi), %edx
6076- movq %r9, 24(%rsp)
6077- movl %edx, 4(%rsp)
6078-
6079- cmpq $0, (%r13)
6080- movq $-ETIMEDOUT, %r14
6081- js 36f
6082-
6083-38: movl cond_futex(%rdi), %r12d
6084-
6085- /* Unlock. */
6086- LOCK
6087-#if cond_lock == 0
6088- decl (%rdi)
6089-#else
6090- decl cond_lock(%rdi)
6091-#endif
6092- jne 33f
6093-
6094-.LcleanupSTART1:
6095-34: callq __pthread_enable_asynccancel
6096- movl %eax, (%rsp)
6097-
6098- movq %r13, %r10
6099- movl $FUTEX_WAIT_BITSET, %esi
6100- LP_OP(cmp) $-1, dep_mutex(%rdi)
6101- je 60f
6102-
6103- mov dep_mutex(%rdi), %R8_LP
6104- /* Requeue to a non-robust PI mutex if the PI bit is set and
6105- the robust bit is not set. */
6106- movl MUTEX_KIND(%r8), %eax
6107- andl $(ROBUST_BIT|PI_BIT), %eax
6108- cmpl $PI_BIT, %eax
6109- jne 61f
6110-
6111- movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
6112- xorl %eax, %eax
6113- /* The following only works like this because we only support
6114- two clocks, represented using a single bit. */
6115- testl $1, cond_nwaiters(%rdi)
6116- movl $FUTEX_CLOCK_REALTIME, %edx
6117- cmove %edx, %eax
6118- orl %eax, %esi
6119- movq %r12, %rdx
6120- addq $cond_futex, %rdi
6121- movl $SYS_futex, %eax
6122- syscall
6123-
6124- cmpl $0, %eax
6125- sete %r15b
6126-
6127-#ifdef __ASSUME_REQUEUE_PI
6128- jmp 62f
6129-#else
6130- je 62f
6131-
6132- /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
6133- successfully, it has already locked the mutex for us and the
6134- pi_flag (%r15b) is set to denote that fact. However, if another
6135- thread changed the futex value before we entered the wait, the
6136- syscall may return an EAGAIN and the mutex is not locked. We go
6137- ahead with a success anyway since later we look at the pi_flag to
6138- decide if we got the mutex or not. The sequence numbers then make
6139- sure that only one of the threads actually wake up. We retry using
6140- normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
6141- and PI futexes don't mix.
6142-
6143- Note that we don't check for EAGAIN specifically; we assume that the
6144- only other error the futex function could return is EAGAIN (barring
6145- the ETIMEOUT of course, for the timeout case in futex) since
6146- anything else would mean an error in our function. It is too
6147- expensive to do that check for every call (which is quite common in
6148- case of a large number of threads), so it has been skipped. */
6149- cmpl $-ENOSYS, %eax
6150- jne 62f
6151-
6152- subq $cond_futex, %rdi
6153-#endif
6154-
6155-61: movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi
6156-60: xorb %r15b, %r15b
6157- xorl %eax, %eax
6158- /* The following only works like this because we only support
6159- two clocks, represented using a single bit. */
6160- testl $1, cond_nwaiters(%rdi)
6161- movl $FUTEX_CLOCK_REALTIME, %edx
6162- movl $0xffffffff, %r9d
6163- cmove %edx, %eax
6164- orl %eax, %esi
6165- movq %r12, %rdx
6166- addq $cond_futex, %rdi
6167- movl $SYS_futex, %eax
6168- syscall
6169-62: movq %rax, %r14
6170-
6171- movl (%rsp), %edi
6172- callq __pthread_disable_asynccancel
6173-.LcleanupEND1:
6174-
6175- /* Lock. */
6176- movq 8(%rsp), %rdi
6177- movl $1, %esi
6178- xorl %eax, %eax
6179- LOCK
6180-#if cond_lock == 0
6181- cmpxchgl %esi, (%rdi)
6182-#else
6183- cmpxchgl %esi, cond_lock(%rdi)
6184-#endif
6185- jne 35f
6186-
6187-36: movl broadcast_seq(%rdi), %edx
6188-
6189- movq woken_seq(%rdi), %rax
6190-
6191- movq wakeup_seq(%rdi), %r9
6192-
6193- cmpl 4(%rsp), %edx
6194- jne 53f
6195-
6196- cmpq 24(%rsp), %r9
6197- jbe 45f
6198-
6199- cmpq %rax, %r9
6200- ja 39f
6201-
6202-45: cmpq $-ETIMEDOUT, %r14
6203- je 99f
6204-
6205- /* We need to go back to futex_wait. If we're using requeue_pi, then
6206- release the mutex we had acquired and go back. */
6207- test %r15b, %r15b
6208- jz 38b
6209-
6210- /* Adjust the mutex values first and then unlock it. The unlock
6211- should always succeed or else the kernel did not lock the
6212- mutex correctly. */
6213- movq %r8, %rdi
6214- callq __pthread_mutex_cond_lock_adjust
6215- xorl %esi, %esi
6216- callq __pthread_mutex_unlock_usercnt
6217- /* Reload cond_var. */
6218- movq 8(%rsp), %rdi
6219- jmp 38b
6220-
6221-99: incq wakeup_seq(%rdi)
6222- incl cond_futex(%rdi)
6223- movl $ETIMEDOUT, %r14d
6224- jmp 44f
6225-
6226-53: xorq %r14, %r14
6227- jmp 54f
6228-
6229-39: xorq %r14, %r14
6230-44: incq woken_seq(%rdi)
6231-
6232-54: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
6233-
6234- /* Wake up a thread which wants to destroy the condvar object. */
6235- cmpq $0xffffffffffffffff, total_seq(%rdi)
6236- jne 55f
6237- movl cond_nwaiters(%rdi), %eax
6238- andl $~((1 << nwaiters_shift) - 1), %eax
6239- jne 55f
6240-
6241- addq $cond_nwaiters, %rdi
6242- LP_OP(cmp) $-1, dep_mutex-cond_nwaiters(%rdi)
6243- movl $1, %edx
6244-#ifdef __ASSUME_PRIVATE_FUTEX
6245- movl $FUTEX_WAKE, %eax
6246- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
6247- cmove %eax, %esi
6248-#else
6249- movl $0, %eax
6250- movl %fs:PRIVATE_FUTEX, %esi
6251- cmove %eax, %esi
6252- orl $FUTEX_WAKE, %esi
6253-#endif
6254- movl $SYS_futex, %eax
6255- syscall
6256- subq $cond_nwaiters, %rdi
6257-
6258-55: LOCK
6259-#if cond_lock == 0
6260- decl (%rdi)
6261-#else
6262- decl cond_lock(%rdi)
6263-#endif
6264- jne 40f
6265-
6266- /* If requeue_pi is used the kernel performs the locking of the
6267- mutex. */
6268-41: movq 16(%rsp), %rdi
6269- testb %r15b, %r15b
6270- jnz 64f
6271-
6272- callq __pthread_mutex_cond_lock
6273-
6274-63: testq %rax, %rax
6275- cmoveq %r14, %rax
6276-
6277-48: addq $FRAME_SIZE, %rsp
6278- cfi_adjust_cfa_offset(-FRAME_SIZE)
6279- popq %r15
6280- cfi_adjust_cfa_offset(-8)
6281- cfi_restore(%r15)
6282- popq %r14
6283- cfi_adjust_cfa_offset(-8)
6284- cfi_restore(%r14)
6285- popq %r13
6286- cfi_adjust_cfa_offset(-8)
6287- cfi_restore(%r13)
6288- popq %r12
6289- cfi_adjust_cfa_offset(-8)
6290- cfi_restore(%r12)
6291-
6292- retq
6293-
6294- cfi_restore_state
6295-
6296-64: callq __pthread_mutex_cond_lock_adjust
6297- movq %r14, %rax
6298- jmp 48b
6299-
6300- /* Initial locking failed. */
6301-31:
6302-#if cond_lock != 0
6303- addq $cond_lock, %rdi
6304-#endif
6305- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6306- movl $LLL_PRIVATE, %eax
6307- movl $LLL_SHARED, %esi
6308- cmovne %eax, %esi
6309- callq __lll_lock_wait
6310- jmp 32b
6311-
6312- /* Unlock in loop requires wakeup. */
6313-33:
6314-#if cond_lock != 0
6315- addq $cond_lock, %rdi
6316-#endif
6317- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6318- movl $LLL_PRIVATE, %eax
6319- movl $LLL_SHARED, %esi
6320- cmovne %eax, %esi
6321- callq __lll_unlock_wake
6322- jmp 34b
6323-
6324- /* Locking in loop failed. */
6325-35:
6326-#if cond_lock != 0
6327- addq $cond_lock, %rdi
6328-#endif
6329- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6330- movl $LLL_PRIVATE, %eax
6331- movl $LLL_SHARED, %esi
6332- cmovne %eax, %esi
6333- callq __lll_lock_wait
6334-#if cond_lock != 0
6335- subq $cond_lock, %rdi
6336-#endif
6337- jmp 36b
6338-
6339- /* Unlock after loop requires wakeup. */
6340-40:
6341-#if cond_lock != 0
6342- addq $cond_lock, %rdi
6343-#endif
6344- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6345- movl $LLL_PRIVATE, %eax
6346- movl $LLL_SHARED, %esi
6347- cmovne %eax, %esi
6348- callq __lll_unlock_wake
6349- jmp 41b
6350-
6351- /* The initial unlocking of the mutex failed. */
6352-46: movq 8(%rsp), %rdi
6353- movq %rax, (%rsp)
6354- LOCK
6355-#if cond_lock == 0
6356- decl (%rdi)
6357-#else
6358- decl cond_lock(%rdi)
6359-#endif
6360- jne 47f
6361-
6362-#if cond_lock != 0
6363- addq $cond_lock, %rdi
6364-#endif
6365- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6366- movl $LLL_PRIVATE, %eax
6367- movl $LLL_SHARED, %esi
6368- cmovne %eax, %esi
6369- callq __lll_unlock_wake
6370-
6371-47: movq (%rsp), %rax
6372- jmp 48b
6373-
6374- .size __pthread_cond_timedwait, .-__pthread_cond_timedwait
6375-versioned_symbol (libpthread, __pthread_cond_timedwait, pthread_cond_timedwait,
6376- GLIBC_2_3_2)
6377-
6378-
6379- .align 16
6380- .type __condvar_cleanup2, @function
6381-__condvar_cleanup2:
6382- /* Stack frame:
6383-
6384- rsp + 72
6385- +--------------------------+
6386- rsp + 64 | %r12 |
6387- +--------------------------+
6388- rsp + 56 | %r13 |
6389- +--------------------------+
6390- rsp + 48 | %r14 |
6391- +--------------------------+
6392- rsp + 24 | unused |
6393- +--------------------------+
6394- rsp + 16 | mutex pointer |
6395- +--------------------------+
6396- rsp + 8 | condvar pointer |
6397- +--------------------------+
6398- rsp + 4 | old broadcast_seq value |
6399- +--------------------------+
6400- rsp + 0 | old cancellation mode |
6401- +--------------------------+
6402- */
6403-
6404- movq %rax, 24(%rsp)
6405-
6406- /* Get internal lock. */
6407- movq 8(%rsp), %rdi
6408- movl $1, %esi
6409- xorl %eax, %eax
6410- LOCK
6411-#if cond_lock == 0
6412- cmpxchgl %esi, (%rdi)
6413-#else
6414- cmpxchgl %esi, cond_lock(%rdi)
6415-#endif
6416- jz 1f
6417-
6418-#if cond_lock != 0
6419- addq $cond_lock, %rdi
6420-#endif
6421- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6422- movl $LLL_PRIVATE, %eax
6423- movl $LLL_SHARED, %esi
6424- cmovne %eax, %esi
6425- callq __lll_lock_wait
6426-#if cond_lock != 0
6427- subq $cond_lock, %rdi
6428-#endif
6429-
6430-1: movl broadcast_seq(%rdi), %edx
6431- cmpl 4(%rsp), %edx
6432- jne 3f
6433-
6434- /* We increment the wakeup_seq counter only if it is lower than
6435- total_seq. If this is not the case the thread was woken and
6436- then canceled. In this case we ignore the signal. */
6437- movq total_seq(%rdi), %rax
6438- cmpq wakeup_seq(%rdi), %rax
6439- jbe 6f
6440- incq wakeup_seq(%rdi)
6441- incl cond_futex(%rdi)
6442-6: incq woken_seq(%rdi)
6443-
6444-3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
6445-
6446- /* Wake up a thread which wants to destroy the condvar object. */
6447- xorq %r12, %r12
6448- cmpq $0xffffffffffffffff, total_seq(%rdi)
6449- jne 4f
6450- movl cond_nwaiters(%rdi), %eax
6451- andl $~((1 << nwaiters_shift) - 1), %eax
6452- jne 4f
6453-
6454- LP_OP(cmp) $-1, dep_mutex(%rdi)
6455- leaq cond_nwaiters(%rdi), %rdi
6456- movl $1, %edx
6457-#ifdef __ASSUME_PRIVATE_FUTEX
6458- movl $FUTEX_WAKE, %eax
6459- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
6460- cmove %eax, %esi
6461-#else
6462- movl $0, %eax
6463- movl %fs:PRIVATE_FUTEX, %esi
6464- cmove %eax, %esi
6465- orl $FUTEX_WAKE, %esi
6466-#endif
6467- movl $SYS_futex, %eax
6468- syscall
6469- subq $cond_nwaiters, %rdi
6470- movl $1, %r12d
6471-
6472-4: LOCK
6473-#if cond_lock == 0
6474- decl (%rdi)
6475-#else
6476- decl cond_lock(%rdi)
6477-#endif
6478- je 2f
6479-#if cond_lock != 0
6480- addq $cond_lock, %rdi
6481-#endif
6482- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6483- movl $LLL_PRIVATE, %eax
6484- movl $LLL_SHARED, %esi
6485- cmovne %eax, %esi
6486- callq __lll_unlock_wake
6487-
6488- /* Wake up all waiters to make sure no signal gets lost. */
6489-2: testq %r12, %r12
6490- jnz 5f
6491- addq $cond_futex, %rdi
6492- LP_OP(cmp) $-1, dep_mutex-cond_futex(%rdi)
6493- movl $0x7fffffff, %edx
6494-#ifdef __ASSUME_PRIVATE_FUTEX
6495- movl $FUTEX_WAKE, %eax
6496- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
6497- cmove %eax, %esi
6498-#else
6499- movl $0, %eax
6500- movl %fs:PRIVATE_FUTEX, %esi
6501- cmove %eax, %esi
6502- orl $FUTEX_WAKE, %esi
6503-#endif
6504- movl $SYS_futex, %eax
6505- syscall
6506-
6507- /* Lock the mutex only if we don't own it already. This only happens
6508- in case of PI mutexes, if we got cancelled after a successful
6509- return of the futex syscall and before disabling async
6510- cancellation. */
6511-5: movq 16(%rsp), %rdi
6512- movl MUTEX_KIND(%rdi), %eax
6513- andl $(ROBUST_BIT|PI_BIT), %eax
6514- cmpl $PI_BIT, %eax
6515- jne 7f
6516-
6517- movl (%rdi), %eax
6518- andl $TID_MASK, %eax
6519- cmpl %eax, %fs:TID
6520- jne 7f
6521- /* We managed to get the lock. Fix it up before returning. */
6522- callq __pthread_mutex_cond_lock_adjust
6523- jmp 8f
6524-
6525-7: callq __pthread_mutex_cond_lock
6526-
6527-8: movq 24(%rsp), %rdi
6528- movq FRAME_SIZE(%rsp), %r15
6529- movq FRAME_SIZE+8(%rsp), %r14
6530- movq FRAME_SIZE+16(%rsp), %r13
6531- movq FRAME_SIZE+24(%rsp), %r12
6532-.LcallUR:
6533- call _Unwind_Resume
6534- hlt
6535-.LENDCODE:
6536- cfi_endproc
6537- .size __condvar_cleanup2, .-__condvar_cleanup2
6538-
6539-
6540- .section .gcc_except_table,"a",@progbits
6541-.LexceptSTART:
6542- .byte DW_EH_PE_omit # @LPStart format
6543- .byte DW_EH_PE_omit # @TType format
6544- .byte DW_EH_PE_uleb128 # call-site format
6545- .uleb128 .Lcstend-.Lcstbegin
6546-.Lcstbegin:
6547- .uleb128 .LcleanupSTART1-.LSTARTCODE
6548- .uleb128 .LcleanupEND1-.LcleanupSTART1
6549- .uleb128 __condvar_cleanup2-.LSTARTCODE
6550- .uleb128 0
6551- .uleb128 .LcallUR-.LSTARTCODE
6552- .uleb128 .LENDCODE-.LcallUR
6553- .uleb128 0
6554- .uleb128 0
6555-.Lcstend:
6556-
6557-
6558-#ifdef SHARED
6559- .hidden DW.ref.__gcc_personality_v0
6560- .weak DW.ref.__gcc_personality_v0
6561- .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
6562- .align LP_SIZE
6563- .type DW.ref.__gcc_personality_v0, @object
6564- .size DW.ref.__gcc_personality_v0, LP_SIZE
6565-DW.ref.__gcc_personality_v0:
6566- ASM_ADDR __gcc_personality_v0
6567-#endif
6568diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
6569deleted file mode 100644
6570index c82f37b..0000000
6571--- a/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S
6572+++ /dev/null
6573@@ -1,555 +0,0 @@
6574-/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
6575- This file is part of the GNU C Library.
6576- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
6577-
6578- The GNU C Library is free software; you can redistribute it and/or
6579- modify it under the terms of the GNU Lesser General Public
6580- License as published by the Free Software Foundation; either
6581- version 2.1 of the License, or (at your option) any later version.
6582-
6583- The GNU C Library is distributed in the hope that it will be useful,
6584- but WITHOUT ANY WARRANTY; without even the implied warranty of
6585- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
6586- Lesser General Public License for more details.
6587-
6588- You should have received a copy of the GNU Lesser General Public
6589- License along with the GNU C Library; if not, see
6590- <http://www.gnu.org/licenses/>. */
6591-
6592-#include <sysdep.h>
6593-#include <shlib-compat.h>
6594-#include <lowlevellock.h>
6595-#include <lowlevelcond.h>
6596-#include <tcb-offsets.h>
6597-#include <pthread-pi-defines.h>
6598-#include <pthread-errnos.h>
6599-#include <stap-probe.h>
6600-
6601-#include <kernel-features.h>
6602-
6603-
6604- .text
6605-
6606-/* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */
6607- .globl __pthread_cond_wait
6608- .type __pthread_cond_wait, @function
6609- .align 16
6610-__pthread_cond_wait:
6611-.LSTARTCODE:
6612- cfi_startproc
6613-#ifdef SHARED
6614- cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
6615- DW.ref.__gcc_personality_v0)
6616- cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
6617-#else
6618- cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
6619- cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
6620-#endif
6621-
6622-#define FRAME_SIZE (32+8)
6623- leaq -FRAME_SIZE(%rsp), %rsp
6624- cfi_adjust_cfa_offset(FRAME_SIZE)
6625-
6626- /* Stack frame:
6627-
6628- rsp + 32
6629- +--------------------------+
6630- rsp + 24 | old wake_seq value |
6631- +--------------------------+
6632- rsp + 16 | mutex pointer |
6633- +--------------------------+
6634- rsp + 8 | condvar pointer |
6635- +--------------------------+
6636- rsp + 4 | old broadcast_seq value |
6637- +--------------------------+
6638- rsp + 0 | old cancellation mode |
6639- +--------------------------+
6640- */
6641-
6642- LIBC_PROBE (cond_wait, 2, %rdi, %rsi)
6643-
6644- LP_OP(cmp) $-1, dep_mutex(%rdi)
6645-
6646- /* Prepare structure passed to cancellation handler. */
6647- movq %rdi, 8(%rsp)
6648- movq %rsi, 16(%rsp)
6649-
6650- je 15f
6651- mov %RSI_LP, dep_mutex(%rdi)
6652-
6653- /* Get internal lock. */
6654-15: movl $1, %esi
6655- xorl %eax, %eax
6656- LOCK
6657-#if cond_lock == 0
6658- cmpxchgl %esi, (%rdi)
6659-#else
6660- cmpxchgl %esi, cond_lock(%rdi)
6661-#endif
6662- jne 1f
6663-
6664- /* Unlock the mutex. */
6665-2: movq 16(%rsp), %rdi
6666- xorl %esi, %esi
6667- callq __pthread_mutex_unlock_usercnt
6668-
6669- testl %eax, %eax
6670- jne 12f
6671-
6672- movq 8(%rsp), %rdi
6673- incq total_seq(%rdi)
6674- incl cond_futex(%rdi)
6675- addl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
6676-
6677- /* Get and store current wakeup_seq value. */
6678- movq 8(%rsp), %rdi
6679- movq wakeup_seq(%rdi), %r9
6680- movl broadcast_seq(%rdi), %edx
6681- movq %r9, 24(%rsp)
6682- movl %edx, 4(%rsp)
6683-
6684- /* Unlock. */
6685-8: movl cond_futex(%rdi), %edx
6686- LOCK
6687-#if cond_lock == 0
6688- decl (%rdi)
6689-#else
6690- decl cond_lock(%rdi)
6691-#endif
6692- jne 3f
6693-
6694-.LcleanupSTART:
6695-4: callq __pthread_enable_asynccancel
6696- movl %eax, (%rsp)
6697-
6698- xorq %r10, %r10
6699- LP_OP(cmp) $-1, dep_mutex(%rdi)
6700- leaq cond_futex(%rdi), %rdi
6701- movl $FUTEX_WAIT, %esi
6702- je 60f
6703-
6704- mov dep_mutex-cond_futex(%rdi), %R8_LP
6705- /* Requeue to a non-robust PI mutex if the PI bit is set and
6706- the robust bit is not set. */
6707- movl MUTEX_KIND(%r8), %eax
6708- andl $(ROBUST_BIT|PI_BIT), %eax
6709- cmpl $PI_BIT, %eax
6710- jne 61f
6711-
6712- movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi
6713- movl $SYS_futex, %eax
6714- syscall
6715-
6716- cmpl $0, %eax
6717- sete %r8b
6718-
6719-#ifdef __ASSUME_REQUEUE_PI
6720- jmp 62f
6721-#else
6722- je 62f
6723-
6724- /* When a futex syscall with FUTEX_WAIT_REQUEUE_PI returns
6725- successfully, it has already locked the mutex for us and the
6726- pi_flag (%r8b) is set to denote that fact. However, if another
6727- thread changed the futex value before we entered the wait, the
6728- syscall may return an EAGAIN and the mutex is not locked. We go
6729- ahead with a success anyway since later we look at the pi_flag to
6730- decide if we got the mutex or not. The sequence numbers then make
6731- sure that only one of the threads actually wake up. We retry using
6732- normal FUTEX_WAIT only if the kernel returned ENOSYS, since normal
6733- and PI futexes don't mix.
6734-
6735- Note that we don't check for EAGAIN specifically; we assume that the
6736- only other error the futex function could return is EAGAIN since
6737- anything else would mean an error in our function. It is too
6738- expensive to do that check for every call (which is quite common in
6739- case of a large number of threads), so it has been skipped. */
6740- cmpl $-ENOSYS, %eax
6741- jne 62f
6742-
6743-# ifndef __ASSUME_PRIVATE_FUTEX
6744- movl $FUTEX_WAIT, %esi
6745-# endif
6746-#endif
6747-
6748-61:
6749-#ifdef __ASSUME_PRIVATE_FUTEX
6750- movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi
6751-#else
6752- orl %fs:PRIVATE_FUTEX, %esi
6753-#endif
6754-60: xorb %r8b, %r8b
6755- movl $SYS_futex, %eax
6756- syscall
6757-
6758-62: movl (%rsp), %edi
6759- callq __pthread_disable_asynccancel
6760-.LcleanupEND:
6761-
6762- /* Lock. */
6763- movq 8(%rsp), %rdi
6764- movl $1, %esi
6765- xorl %eax, %eax
6766- LOCK
6767-#if cond_lock == 0
6768- cmpxchgl %esi, (%rdi)
6769-#else
6770- cmpxchgl %esi, cond_lock(%rdi)
6771-#endif
6772- jnz 5f
6773-
6774-6: movl broadcast_seq(%rdi), %edx
6775-
6776- movq woken_seq(%rdi), %rax
6777-
6778- movq wakeup_seq(%rdi), %r9
6779-
6780- cmpl 4(%rsp), %edx
6781- jne 16f
6782-
6783- cmpq 24(%rsp), %r9
6784- jbe 19f
6785-
6786- cmpq %rax, %r9
6787- jna 19f
6788-
6789- incq woken_seq(%rdi)
6790-
6791- /* Unlock */
6792-16: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
6793-
6794- /* Wake up a thread which wants to destroy the condvar object. */
6795- cmpq $0xffffffffffffffff, total_seq(%rdi)
6796- jne 17f
6797- movl cond_nwaiters(%rdi), %eax
6798- andl $~((1 << nwaiters_shift) - 1), %eax
6799- jne 17f
6800-
6801- addq $cond_nwaiters, %rdi
6802- LP_OP(cmp) $-1, dep_mutex-cond_nwaiters(%rdi)
6803- movl $1, %edx
6804-#ifdef __ASSUME_PRIVATE_FUTEX
6805- movl $FUTEX_WAKE, %eax
6806- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
6807- cmove %eax, %esi
6808-#else
6809- movl $0, %eax
6810- movl %fs:PRIVATE_FUTEX, %esi
6811- cmove %eax, %esi
6812- orl $FUTEX_WAKE, %esi
6813-#endif
6814- movl $SYS_futex, %eax
6815- syscall
6816- subq $cond_nwaiters, %rdi
6817-
6818-17: LOCK
6819-#if cond_lock == 0
6820- decl (%rdi)
6821-#else
6822- decl cond_lock(%rdi)
6823-#endif
6824- jne 10f
6825-
6826- /* If requeue_pi is used the kernel performs the locking of the
6827- mutex. */
6828-11: movq 16(%rsp), %rdi
6829- testb %r8b, %r8b
6830- jnz 18f
6831-
6832- callq __pthread_mutex_cond_lock
6833-
6834-14: leaq FRAME_SIZE(%rsp), %rsp
6835- cfi_adjust_cfa_offset(-FRAME_SIZE)
6836-
6837- /* We return the result of the mutex_lock operation. */
6838- retq
6839-
6840- cfi_adjust_cfa_offset(FRAME_SIZE)
6841-
6842-18: callq __pthread_mutex_cond_lock_adjust
6843- xorl %eax, %eax
6844- jmp 14b
6845-
6846- /* We need to go back to futex_wait. If we're using requeue_pi, then
6847- release the mutex we had acquired and go back. */
6848-19: testb %r8b, %r8b
6849- jz 8b
6850-
6851- /* Adjust the mutex values first and then unlock it. The unlock
6852- should always succeed or else the kernel did not lock the mutex
6853- correctly. */
6854- movq 16(%rsp), %rdi
6855- callq __pthread_mutex_cond_lock_adjust
6856- movq %rdi, %r8
6857- xorl %esi, %esi
6858- callq __pthread_mutex_unlock_usercnt
6859- /* Reload cond_var. */
6860- movq 8(%rsp), %rdi
6861- jmp 8b
6862-
6863- /* Initial locking failed. */
6864-1:
6865-#if cond_lock != 0
6866- addq $cond_lock, %rdi
6867-#endif
6868- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6869- movl $LLL_PRIVATE, %eax
6870- movl $LLL_SHARED, %esi
6871- cmovne %eax, %esi
6872- callq __lll_lock_wait
6873- jmp 2b
6874-
6875- /* Unlock in loop requires wakeup. */
6876-3:
6877-#if cond_lock != 0
6878- addq $cond_lock, %rdi
6879-#endif
6880- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6881- movl $LLL_PRIVATE, %eax
6882- movl $LLL_SHARED, %esi
6883- cmovne %eax, %esi
6884- /* The call preserves %rdx. */
6885- callq __lll_unlock_wake
6886-#if cond_lock != 0
6887- subq $cond_lock, %rdi
6888-#endif
6889- jmp 4b
6890-
6891- /* Locking in loop failed. */
6892-5:
6893-#if cond_lock != 0
6894- addq $cond_lock, %rdi
6895-#endif
6896- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6897- movl $LLL_PRIVATE, %eax
6898- movl $LLL_SHARED, %esi
6899- cmovne %eax, %esi
6900- callq __lll_lock_wait
6901-#if cond_lock != 0
6902- subq $cond_lock, %rdi
6903-#endif
6904- jmp 6b
6905-
6906- /* Unlock after loop requires wakeup. */
6907-10:
6908-#if cond_lock != 0
6909- addq $cond_lock, %rdi
6910-#endif
6911- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6912- movl $LLL_PRIVATE, %eax
6913- movl $LLL_SHARED, %esi
6914- cmovne %eax, %esi
6915- callq __lll_unlock_wake
6916- jmp 11b
6917-
6918- /* The initial unlocking of the mutex failed. */
6919-12: movq %rax, %r10
6920- movq 8(%rsp), %rdi
6921- LOCK
6922-#if cond_lock == 0
6923- decl (%rdi)
6924-#else
6925- decl cond_lock(%rdi)
6926-#endif
6927- je 13f
6928-
6929-#if cond_lock != 0
6930- addq $cond_lock, %rdi
6931-#endif
6932- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6933- movl $LLL_PRIVATE, %eax
6934- movl $LLL_SHARED, %esi
6935- cmovne %eax, %esi
6936- callq __lll_unlock_wake
6937-
6938-13: movq %r10, %rax
6939- jmp 14b
6940-
6941- .size __pthread_cond_wait, .-__pthread_cond_wait
6942-versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
6943- GLIBC_2_3_2)
6944-
6945-
6946- .align 16
6947- .type __condvar_cleanup1, @function
6948- .globl __condvar_cleanup1
6949- .hidden __condvar_cleanup1
6950-__condvar_cleanup1:
6951- /* Stack frame:
6952-
6953- rsp + 32
6954- +--------------------------+
6955- rsp + 24 | unused |
6956- +--------------------------+
6957- rsp + 16 | mutex pointer |
6958- +--------------------------+
6959- rsp + 8 | condvar pointer |
6960- +--------------------------+
6961- rsp + 4 | old broadcast_seq value |
6962- +--------------------------+
6963- rsp + 0 | old cancellation mode |
6964- +--------------------------+
6965- */
6966-
6967- movq %rax, 24(%rsp)
6968-
6969- /* Get internal lock. */
6970- movq 8(%rsp), %rdi
6971- movl $1, %esi
6972- xorl %eax, %eax
6973- LOCK
6974-#if cond_lock == 0
6975- cmpxchgl %esi, (%rdi)
6976-#else
6977- cmpxchgl %esi, cond_lock(%rdi)
6978-#endif
6979- jz 1f
6980-
6981-#if cond_lock != 0
6982- addq $cond_lock, %rdi
6983-#endif
6984- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
6985- movl $LLL_PRIVATE, %eax
6986- movl $LLL_SHARED, %esi
6987- cmovne %eax, %esi
6988- callq __lll_lock_wait
6989-#if cond_lock != 0
6990- subq $cond_lock, %rdi
6991-#endif
6992-
6993-1: movl broadcast_seq(%rdi), %edx
6994- cmpl 4(%rsp), %edx
6995- jne 3f
6996-
6997- /* We increment the wakeup_seq counter only if it is lower than
6998- total_seq. If this is not the case the thread was woken and
6999- then canceled. In this case we ignore the signal. */
7000- movq total_seq(%rdi), %rax
7001- cmpq wakeup_seq(%rdi), %rax
7002- jbe 6f
7003- incq wakeup_seq(%rdi)
7004- incl cond_futex(%rdi)
7005-6: incq woken_seq(%rdi)
7006-
7007-3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi)
7008-
7009- /* Wake up a thread which wants to destroy the condvar object. */
7010- xorl %ecx, %ecx
7011- cmpq $0xffffffffffffffff, total_seq(%rdi)
7012- jne 4f
7013- movl cond_nwaiters(%rdi), %eax
7014- andl $~((1 << nwaiters_shift) - 1), %eax
7015- jne 4f
7016-
7017- LP_OP(cmp) $-1, dep_mutex(%rdi)
7018- leaq cond_nwaiters(%rdi), %rdi
7019- movl $1, %edx
7020-#ifdef __ASSUME_PRIVATE_FUTEX
7021- movl $FUTEX_WAKE, %eax
7022- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
7023- cmove %eax, %esi
7024-#else
7025- movl $0, %eax
7026- movl %fs:PRIVATE_FUTEX, %esi
7027- cmove %eax, %esi
7028- orl $FUTEX_WAKE, %esi
7029-#endif
7030- movl $SYS_futex, %eax
7031- syscall
7032- subq $cond_nwaiters, %rdi
7033- movl $1, %ecx
7034-
7035-4: LOCK
7036-#if cond_lock == 0
7037- decl (%rdi)
7038-#else
7039- decl cond_lock(%rdi)
7040-#endif
7041- je 2f
7042-#if cond_lock != 0
7043- addq $cond_lock, %rdi
7044-#endif
7045- LP_OP(cmp) $-1, dep_mutex-cond_lock(%rdi)
7046- movl $LLL_PRIVATE, %eax
7047- movl $LLL_SHARED, %esi
7048- cmovne %eax, %esi
7049- /* The call preserves %rcx. */
7050- callq __lll_unlock_wake
7051-
7052- /* Wake up all waiters to make sure no signal gets lost. */
7053-2: testl %ecx, %ecx
7054- jnz 5f
7055- addq $cond_futex, %rdi
7056- LP_OP(cmp) $-1, dep_mutex-cond_futex(%rdi)
7057- movl $0x7fffffff, %edx
7058-#ifdef __ASSUME_PRIVATE_FUTEX
7059- movl $FUTEX_WAKE, %eax
7060- movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi
7061- cmove %eax, %esi
7062-#else
7063- movl $0, %eax
7064- movl %fs:PRIVATE_FUTEX, %esi
7065- cmove %eax, %esi
7066- orl $FUTEX_WAKE, %esi
7067-#endif
7068- movl $SYS_futex, %eax
7069- syscall
7070-
7071- /* Lock the mutex only if we don't own it already. This only happens
7072- in case of PI mutexes, if we got cancelled after a successful
7073- return of the futex syscall and before disabling async
7074- cancellation. */
7075-5: movq 16(%rsp), %rdi
7076- movl MUTEX_KIND(%rdi), %eax
7077- andl $(ROBUST_BIT|PI_BIT), %eax
7078- cmpl $PI_BIT, %eax
7079- jne 7f
7080-
7081- movl (%rdi), %eax
7082- andl $TID_MASK, %eax
7083- cmpl %eax, %fs:TID
7084- jne 7f
7085- /* We managed to get the lock. Fix it up before returning. */
7086- callq __pthread_mutex_cond_lock_adjust
7087- jmp 8f
7088-
7089-
7090-7: callq __pthread_mutex_cond_lock
7091-
7092-8: movq 24(%rsp), %rdi
7093-.LcallUR:
7094- call _Unwind_Resume
7095- hlt
7096-.LENDCODE:
7097- cfi_endproc
7098- .size __condvar_cleanup1, .-__condvar_cleanup1
7099-
7100-
7101- .section .gcc_except_table,"a",@progbits
7102-.LexceptSTART:
7103- .byte DW_EH_PE_omit # @LPStart format
7104- .byte DW_EH_PE_omit # @TType format
7105- .byte DW_EH_PE_uleb128 # call-site format
7106- .uleb128 .Lcstend-.Lcstbegin
7107-.Lcstbegin:
7108- .uleb128 .LcleanupSTART-.LSTARTCODE
7109- .uleb128 .LcleanupEND-.LcleanupSTART
7110- .uleb128 __condvar_cleanup1-.LSTARTCODE
7111- .uleb128 0
7112- .uleb128 .LcallUR-.LSTARTCODE
7113- .uleb128 .LENDCODE-.LcallUR
7114- .uleb128 0
7115- .uleb128 0
7116-.Lcstend:
7117-
7118-
7119-#ifdef SHARED
7120- .hidden DW.ref.__gcc_personality_v0
7121- .weak DW.ref.__gcc_personality_v0
7122- .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
7123- .align LP_SIZE
7124- .type DW.ref.__gcc_personality_v0, @object
7125- .size DW.ref.__gcc_personality_v0, LP_SIZE
7126-DW.ref.__gcc_personality_v0:
7127- ASM_ADDR __gcc_personality_v0
7128-#endif
7129diff --git a/sysdeps/x86/bits/pthreadtypes.h b/sysdeps/x86/bits/pthreadtypes.h
7130index 16b8f4f..a3a738f 100644
7131--- a/sysdeps/x86/bits/pthreadtypes.h
7132+++ b/sysdeps/x86/bits/pthreadtypes.h
7133@@ -140,14 +140,27 @@ typedef union
7134 {
7135 struct
7136 {
7137- int __lock;
7138- unsigned int __futex;
7139- __extension__ unsigned long long int __total_seq;
7140- __extension__ unsigned long long int __wakeup_seq;
7141- __extension__ unsigned long long int __woken_seq;
7142- void *__mutex;
7143- unsigned int __nwaiters;
7144- unsigned int __broadcast_seq;
7145+ __extension__ union
7146+ {
7147+ __extension__ unsigned long long int __wseq;
7148+ struct {
7149+ unsigned int __low;
7150+ unsigned int __high;
7151+ } __wseq32;
7152+ };
7153+ __extension__ union
7154+ {
7155+ __extension__ unsigned long long int __g1_start;
7156+ struct {
7157+ unsigned int __low;
7158+ unsigned int __high;
7159+ } __g1_start32;
7160+ };
7161+ unsigned int __g_refs[2];
7162+ unsigned int __g_size[2];
7163+ unsigned int __g1_orig_size;
7164+ unsigned int __wrefs;
7165+ unsigned int __g_signals[2];
7166 } __data;
7167 char __size[__SIZEOF_PTHREAD_COND_T];
7168 __extension__ long long int __align;
7169--
71702.10.2
7171
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-05-20 23:49:43 +0000