| Message ID | 20190222192703.18177-4-adhemerval.zanella@linaro.org |
|---|---|
| State | New |
| Headers | show |
| Series | [1/4] Add single-thread.h header | expand |
Ping. On 22/02/2019 16:27, Adhemerval Zanella wrote: > This patch removes the arch-specific x86 assembly implementation for > low level locking and consolidate both 64 bits and 32 bits in a > single implementation. > > Different than other architectures, x86 lll_trylock, lll_lock, and > lll_unlock implements a single-thread optimization to avoid atomic > operation, using cmpxchgl instead. This patch implements by using > the new single-thread.h definitions in a generic way, although using > the previous semantic. > > The lll_cond_trylock, lll_cond_lock, and lll_timedlock just use > atomic operations plus calls to lll_lock_wait*. > > For __lll_lock_wait_private and __lll_lock_wait the generic implemtation > there is no indication that assembly implementation is required > performance-wise. > > Checked on x86_64-linux-gnu and i686-linux-gnu. > > * sysdeps/nptl/lowlevellock.h (__lll_trylock): New macro. > (lll_trylock): Call __lll_trylock. > * sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S: Remove file. > * sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c: Likewise. > * sysdeps/unix/sysv/linux/i386/lowlevellock.S: Likewise. > * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise. > * sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Likewise. > * sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c: Likewise. > * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise. > * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. > * sysdeps/unix/sysv/linux/x86/lowlevellock.h: New file. > * sysdeps/unix/sysv/linux/x86_64/cancellation.S: Include > lowlevellock-futex.h. > --- > sysdeps/nptl/lowlevellock.h | 4 +- > .../unix/sysv/linux/i386/libc-lowlevellock.S | 19 - > .../unix/sysv/linux/i386/lll_timedlock_wait.c | 1 - > sysdeps/unix/sysv/linux/i386/lowlevellock.S | 368 ------------------ > sysdeps/unix/sysv/linux/i386/lowlevellock.h | 240 ------------ > sysdeps/unix/sysv/linux/x86/lowlevellock.h | 110 ++++++ > sysdeps/unix/sysv/linux/x86_64/cancellation.S | 2 +- > .../sysv/linux/x86_64/libc-lowlevellock.S | 19 - > .../sysv/linux/x86_64/lll_timedlock_wait.c | 1 - > sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 348 ----------------- > sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 243 ------------ > 11 files changed, 114 insertions(+), 1241 deletions(-) > delete mode 100644 sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c > delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.h > create mode 100644 sysdeps/unix/sysv/linux/x86/lowlevellock.h > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.h > > diff --git a/sysdeps/nptl/lowlevellock.h b/sysdeps/nptl/lowlevellock.h > index 6f017afdd5..e905829ee4 100644 > --- a/sysdeps/nptl/lowlevellock.h > +++ b/sysdeps/nptl/lowlevellock.h > @@ -63,8 +63,10 @@ > /* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return > 0. Otherwise leave lock unchanged and return non-zero to indicate that the > lock was not acquired. */ > +#define __lll_trylock(lock) \ > + __glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0)) > #define lll_trylock(lock) \ > - __glibc_unlikely (atomic_compare_and_exchange_bool_acq (&(lock), 1, 0)) > + __lll_trylock (&(lock)) > > /* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and > return 0. Otherwise leave lock unchanged and return non-zero to indicate > diff --git a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S > deleted file mode 100644 > index 32025277f3..0000000000 > --- a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S > +++ /dev/null > @@ -1,19 +0,0 @@ > -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <http://www.gnu.org/licenses/>. */ > - > -#include "lowlevellock.S" > diff --git a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c > deleted file mode 100644 > index f6875b8f89..0000000000 > --- a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c > +++ /dev/null > @@ -1 +0,0 @@ > -/* __lll_timedlock_wait is in lowlevellock.S. */ > diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.S b/sysdeps/unix/sysv/linux/i386/lowlevellock.S > deleted file mode 100644 > index 83191a3899..0000000000 > --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.S > +++ /dev/null > @@ -1,368 +0,0 @@ > -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <http://www.gnu.org/licenses/>. */ > - > -#include <sysdep.h> > -#include <pthread-errnos.h> > -#include <kernel-features.h> > -#include <lowlevellock.h> > - > -#include <stap-probe.h> > - > - .text > - > -#define LOAD_PRIVATE_FUTEX_WAIT(reg) \ > - movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg > -#define LOAD_PRIVATE_FUTEX_WAKE(reg) \ > - movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg > -#define LOAD_FUTEX_WAIT(reg) \ > - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg > -#define LOAD_FUTEX_WAIT_ABS(reg) \ > - xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg > -#define LOAD_FUTEX_WAKE(reg) \ > - xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg > - > - .globl __lll_lock_wait_private > - .type __lll_lock_wait_private,@function > - .hidden __lll_lock_wait_private > - .align 16 > -__lll_lock_wait_private: > - cfi_startproc > - pushl %edx > - cfi_adjust_cfa_offset(4) > - pushl %ebx > - cfi_adjust_cfa_offset(4) > - pushl %esi > - cfi_adjust_cfa_offset(4) > - cfi_offset(%edx, -8) > - cfi_offset(%ebx, -12) > - cfi_offset(%esi, -16) > - > - movl $2, %edx > - movl %ecx, %ebx > - xorl %esi, %esi /* No timeout. */ > - LOAD_PRIVATE_FUTEX_WAIT (%ecx) > - > - cmpl %edx, %eax /* NB: %edx == 2 */ > - jne 2f > - > -1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx) > - movl $SYS_futex, %eax > - ENTER_KERNEL > - > -2: movl %edx, %eax > - xchgl %eax, (%ebx) /* NB: lock is implied */ > - > - testl %eax, %eax > - jnz 1b > - > - popl %esi > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%esi) > - popl %ebx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebx) > - popl %edx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%edx) > - ret > - cfi_endproc > - .size __lll_lock_wait_private,.-__lll_lock_wait_private > - > -#if !IS_IN (libc) > - .globl __lll_lock_wait > - .type __lll_lock_wait,@function > - .hidden __lll_lock_wait > - .align 16 > -__lll_lock_wait: > - cfi_startproc > - pushl %edx > - cfi_adjust_cfa_offset(4) > - pushl %ebx > - cfi_adjust_cfa_offset(4) > - pushl %esi > - cfi_adjust_cfa_offset(4) > - cfi_offset(%edx, -8) > - cfi_offset(%ebx, -12) > - cfi_offset(%esi, -16) > - > - movl %edx, %ebx > - movl $2, %edx > - xorl %esi, %esi /* No timeout. */ > - LOAD_FUTEX_WAIT (%ecx) > - > - cmpl %edx, %eax /* NB: %edx == 2 */ > - jne 2f > - > -1: movl $SYS_futex, %eax > - ENTER_KERNEL > - > -2: movl %edx, %eax > - xchgl %eax, (%ebx) /* NB: lock is implied */ > - > - testl %eax, %eax > - jnz 1b > - > - popl %esi > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%esi) > - popl %ebx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebx) > - popl %edx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%edx) > - ret > - cfi_endproc > - .size __lll_lock_wait,.-__lll_lock_wait > - > - /* %ecx: futex > - %esi: flags > - %edx: timeout > - %eax: futex value > - */ > - .globl __lll_timedlock_wait > - .type __lll_timedlock_wait,@function > - .hidden __lll_timedlock_wait > - .align 16 > -__lll_timedlock_wait: > - cfi_startproc > - pushl %ebp > - cfi_adjust_cfa_offset(4) > - cfi_rel_offset(%ebp, 0) > - pushl %ebx > - cfi_adjust_cfa_offset(4) > - cfi_rel_offset(%ebx, 0) > - > -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME > -# ifdef PIC > - LOAD_PIC_REG (bx) > - cmpl $0, __have_futex_clock_realtime@GOTOFF(%ebx) > -# else > - cmpl $0, __have_futex_clock_realtime > -# endif > - je .Lreltmo > -# endif > - > - cmpl $0, (%edx) > - js 8f > - > - movl %ecx, %ebx > - movl %esi, %ecx > - movl %edx, %esi > - movl $0xffffffff, %ebp > - LOAD_FUTEX_WAIT_ABS (%ecx) > - > - movl $2, %edx > - cmpl %edx, %eax > - jne 2f > - > -1: movl $SYS_futex, %eax > - movl $2, %edx > - ENTER_KERNEL > - > -2: xchgl %edx, (%ebx) /* NB: lock is implied */ > - > - testl %edx, %edx > - jz 3f > - > - cmpl $-ETIMEDOUT, %eax > - je 4f > - cmpl $-EINVAL, %eax > - jne 1b > -4: movl %eax, %edx > - negl %edx > - > -3: movl %edx, %eax > -7: popl %ebx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebx) > - popl %ebp > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebp) > - ret > - > -8: movl $ETIMEDOUT, %eax > - jmp 7b > - > -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME > -.Lreltmo: > - /* Check for a valid timeout value. */ > - cmpl $1000000000, 4(%edx) > - jae 3f > - > - pushl %esi > - cfi_adjust_cfa_offset(4) > - cfi_rel_offset(%esi, 0) > - pushl %edi > - cfi_adjust_cfa_offset(4) > - cfi_rel_offset(%edi, 0) > - > - /* Stack frame for the timespec and timeval structs. */ > - subl $8, %esp > - cfi_adjust_cfa_offset(8) > - > - movl %ecx, %ebp > - movl %edx, %edi > - > - movl $2, %edx > - xchgl %edx, (%ebp) > - > - test %edx, %edx > - je 6f > - > -1: > - /* Get current time. */ > - movl %esp, %ebx > - xorl %ecx, %ecx > - movl $__NR_gettimeofday, %eax > - ENTER_KERNEL > - > - /* Compute relative timeout. */ > - movl 4(%esp), %eax > - movl $1000, %edx > - mul %edx /* Milli seconds to nano seconds. */ > - movl (%edi), %ecx > - movl 4(%edi), %edx > - subl (%esp), %ecx > - subl %eax, %edx > - jns 4f > - addl $1000000000, %edx > - subl $1, %ecx > -4: testl %ecx, %ecx > - js 2f /* Time is already up. */ > - > - /* Store relative timeout. */ > - movl %ecx, (%esp) > - movl %edx, 4(%esp) > - > - /* Futex call. */ > - movl %ebp, %ebx > - movl $2, %edx > - movl %esp, %esi > - movl 16(%esp), %ecx > - LOAD_FUTEX_WAIT (%ecx) > - movl $SYS_futex, %eax > - ENTER_KERNEL > - > - /* NB: %edx == 2 */ > - xchgl %edx, (%ebp) > - > - testl %edx, %edx > - je 6f > - > - cmpl $-ETIMEDOUT, %eax > - jne 1b > -2: movl $ETIMEDOUT, %edx > - > -6: addl $8, %esp > - cfi_adjust_cfa_offset(-8) > - popl %edi > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%edi) > - popl %esi > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%esi) > -7: popl %ebx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebx) > - popl %ebp > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebp) > - movl %edx, %eax > - ret > - > -3: movl $EINVAL, %edx > - jmp 7b > -# endif > - cfi_endproc > - .size __lll_timedlock_wait,.-__lll_timedlock_wait > -#endif > - > - .globl __lll_unlock_wake_private > - .type __lll_unlock_wake_private,@function > - .hidden __lll_unlock_wake_private > - .align 16 > -__lll_unlock_wake_private: > - cfi_startproc > - pushl %ebx > - cfi_adjust_cfa_offset(4) > - pushl %ecx > - cfi_adjust_cfa_offset(4) > - pushl %edx > - cfi_adjust_cfa_offset(4) > - cfi_offset(%ebx, -8) > - cfi_offset(%ecx, -12) > - cfi_offset(%edx, -16) > - > - movl %eax, %ebx > - movl $0, (%eax) > - LOAD_PRIVATE_FUTEX_WAKE (%ecx) > - movl $1, %edx /* Wake one thread. */ > - movl $SYS_futex, %eax > - ENTER_KERNEL > - > - popl %edx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%edx) > - popl %ecx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ecx) > - popl %ebx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebx) > - ret > - cfi_endproc > - .size __lll_unlock_wake_private,.-__lll_unlock_wake_private > - > -#if !IS_IN (libc) > - .globl __lll_unlock_wake > - .type __lll_unlock_wake,@function > - .hidden __lll_unlock_wake > - .align 16 > -__lll_unlock_wake: > - cfi_startproc > - pushl %ebx > - cfi_adjust_cfa_offset(4) > - pushl %ecx > - cfi_adjust_cfa_offset(4) > - pushl %edx > - cfi_adjust_cfa_offset(4) > - cfi_offset(%ebx, -8) > - cfi_offset(%ecx, -12) > - cfi_offset(%edx, -16) > - > - movl %eax, %ebx > - movl $0, (%eax) > - LOAD_FUTEX_WAKE (%ecx) > - movl $1, %edx /* Wake one thread. */ > - movl $SYS_futex, %eax > - ENTER_KERNEL > - > - popl %edx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%edx) > - popl %ecx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ecx) > - popl %ebx > - cfi_adjust_cfa_offset(-4) > - cfi_restore(%ebx) > - ret > - cfi_endproc > - .size __lll_unlock_wake,.-__lll_unlock_wake > -#endif > diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/sysdeps/unix/sysv/linux/i386/lowlevellock.h > deleted file mode 100644 > index 94dccc4ce7..0000000000 > --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.h > +++ /dev/null > @@ -1,240 +0,0 @@ > -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <http://www.gnu.org/licenses/>. */ > - > -#ifndef _LOWLEVELLOCK_H > -#define _LOWLEVELLOCK_H 1 > - > -#ifndef __ASSEMBLER__ > -# include <time.h> > -# include <sys/param.h> > -# include <bits/pthreadtypes.h> > -# include <kernel-features.h> > -/* <tcb-offsets.h> is generated from tcb-offsets.sym to define offsets > - and sizes of types in <tls.h> as well as <pthread.h> which includes > - <lowlevellock.h> via nptl/descr.h. Don't include <tcb-offsets.h> > - when generating <tcb-offsets.h> to avoid circular dependency which > - may lead to build hang on a many-core machine. */ > -# ifndef GEN_AS_CONST_HEADERS > -# include <tcb-offsets.h> > -# endif > - > -# ifndef LOCK_INSTR > -# ifdef UP > -# define LOCK_INSTR /* nothing */ > -# else > -# define LOCK_INSTR "lock;" > -# endif > -# endif > -#else > -# ifndef LOCK > -# ifdef UP > -# define LOCK > -# else > -# define LOCK lock > -# endif > -# endif > -#endif > - > -#include <lowlevellock-futex.h> > - > -/* XXX Remove when no assembler code uses futexes anymore. */ > -#define SYS_futex __NR_futex > - > -#ifndef __ASSEMBLER__ > - > -/* Initializer for compatibility lock. */ > -#define LLL_LOCK_INITIALIZER (0) > -#define LLL_LOCK_INITIALIZER_LOCKED (1) > -#define LLL_LOCK_INITIALIZER_WAITERS (2) > - > - > -/* NB: in the lll_trylock macro we simply return the value in %eax > - after the cmpxchg instruction. In case the operation succeded this > - value is zero. In case the operation failed, the cmpxchg instruction > - has loaded the current value of the memory work which is guaranteed > - to be nonzero. */ > -#if !IS_IN (libc) || defined UP > -# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" > -#else > -# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \ > - "je 0f\n\t" \ > - "lock\n" \ > - "0:\tcmpxchgl %2, %1" > -#endif > - > -#define lll_trylock(futex) \ > - ({ int ret; \ > - __asm __volatile (__lll_trylock_asm \ > - : "=a" (ret), "=m" (futex) \ > - : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ > - "0" (LLL_LOCK_INITIALIZER), \ > - "i" (MULTIPLE_THREADS_OFFSET) \ > - : "memory"); \ > - ret; }) > - > - > -#define lll_cond_trylock(futex) \ > - ({ int ret; \ > - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ > - : "=a" (ret), "=m" (futex) \ > - : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ > - "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ > - : "memory"); \ > - ret; }) > - > -#if !IS_IN (libc) || defined UP > -# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t" > -#else > -# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \ > - "je 0f\n\t" \ > - "lock\n" \ > - "0:\tcmpxchgl %1, %2\n\t" > -#endif > - > -#define lll_lock(futex, private) \ > - (void) \ > - ({ int ignore1, ignore2; \ > - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ > - __asm __volatile (__lll_lock_asm_start \ > - "jz 18f\n\t" \ > - "1:\tleal %2, %%ecx\n" \ > - "2:\tcall __lll_lock_wait_private\n" \ > - "18:" \ > - : "=a" (ignore1), "=c" (ignore2), "=m" (futex) \ > - : "0" (0), "1" (1), "m" (futex), \ > - "i" (MULTIPLE_THREADS_OFFSET) \ > - : "memory"); \ > - else \ > - { \ > - int ignore3; \ > - __asm __volatile (__lll_lock_asm_start \ > - "jz 18f\n\t" \ > - "1:\tleal %2, %%edx\n" \ > - "0:\tmovl %8, %%ecx\n" \ > - "2:\tcall __lll_lock_wait\n" \ > - "18:" \ > - : "=a" (ignore1), "=c" (ignore2), \ > - "=m" (futex), "=&d" (ignore3) \ > - : "1" (1), "m" (futex), \ > - "i" (MULTIPLE_THREADS_OFFSET), "0" (0), \ > - "g" ((int) (private)) \ > - : "memory"); \ > - } \ > - }) > - > - > -/* Special version of lll_lock which causes the unlock function to > - always wakeup waiters. */ > -#define lll_cond_lock(futex, private) \ > - (void) \ > - ({ int ignore1, ignore2, ignore3; \ > - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ > - "jz 18f\n\t" \ > - "1:\tleal %2, %%edx\n" \ > - "0:\tmovl %7, %%ecx\n" \ > - "2:\tcall __lll_lock_wait\n" \ > - "18:" \ > - : "=a" (ignore1), "=c" (ignore2), "=m" (futex), \ > - "=&d" (ignore3) \ > - : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\ > - : "memory"); \ > - }) > - > - > -#define lll_timedlock(futex, timeout, private) \ > - ({ int result, ignore1, ignore2, ignore3; \ > - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ > - "jz 18f\n\t" \ > - "1:\tleal %3, %%ecx\n" \ > - "0:\tmovl %8, %%edx\n" \ > - "2:\tcall __lll_timedlock_wait\n" \ > - "18:" \ > - : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \ > - "=m" (futex), "=S" (ignore3) \ > - : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ > - "4" ((int) (private)) \ > - : "memory"); \ > - result; }) > - > -extern int __lll_timedlock_elision (int *futex, short *adapt_count, > - const struct timespec *timeout, > - int private) attribute_hidden; > - > -#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ > - __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) > - > -#if !IS_IN (libc) || defined UP > -# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t" > -#else > -# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \ > - "je 0f\n\t" \ > - "lock\n" \ > - "0:\tsubl $1,%0\n\t" > -#endif > - > -#define lll_unlock(futex, private) \ > - (void) \ > - ({ int ignore; \ > - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ > - __asm __volatile (__lll_unlock_asm \ > - "je 18f\n\t" \ > - "1:\tleal %0, %%eax\n" \ > - "2:\tcall __lll_unlock_wake_private\n" \ > - "18:" \ > - : "=m" (futex), "=&a" (ignore) \ > - : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \ > - : "memory"); \ > - else \ > - { \ > - int ignore2; \ > - __asm __volatile (__lll_unlock_asm \ > - "je 18f\n\t" \ > - "1:\tleal %0, %%eax\n" \ > - "0:\tmovl %5, %%ecx\n" \ > - "2:\tcall __lll_unlock_wake\n" \ > - "18:" \ > - : "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \ > - : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \ > - "g" ((int) (private)) \ > - : "memory"); \ > - } \ > - }) > - > - > -#define lll_islocked(futex) \ > - (futex != LLL_LOCK_INITIALIZER) > - > -extern int __lll_lock_elision (int *futex, short *adapt_count, int private) > - attribute_hidden; > - > -extern int __lll_unlock_elision(int *lock, int private) > - attribute_hidden; > - > -extern int __lll_trylock_elision(int *lock, short *adapt_count) > - attribute_hidden; > - > -#define lll_lock_elision(futex, adapt_count, private) \ > - __lll_lock_elision (&(futex), &(adapt_count), private) > -#define lll_unlock_elision(futex, adapt_count, private) \ > - __lll_unlock_elision (&(futex), private) > -#define lll_trylock_elision(futex, adapt_count) \ > - __lll_trylock_elision(&(futex), &(adapt_count)) > - > -#endif /* !__ASSEMBLER__ */ > - > -#endif /* lowlevellock.h */ > diff --git a/sysdeps/unix/sysv/linux/x86/lowlevellock.h b/sysdeps/unix/sysv/linux/x86/lowlevellock.h > new file mode 100644 > index 0000000000..2652a00736 > --- /dev/null > +++ b/sysdeps/unix/sysv/linux/x86/lowlevellock.h > @@ -0,0 +1,110 @@ > +/* Low-level lock implementation, x86 version. > + Copyright (C) 2019 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <http://www.gnu.org/licenses/>. */ > + > +#ifndef _X86_64_LOWLEVELLOCK_H > +#define _X86_64_LOWLEVELLOCK_H 1 > + > +#ifndef __ASSEMBLER__ > +#include <sysdeps/nptl/lowlevellock.h> > +#include <single-thread.h> > + > +/* The lll_trylock, lll_lock, and lll_unlock implements a single-thread > + optimization using the cmpxchgl instruction. It checks if the process > + is single thread and avoid a more expensive atomic instruction. */ > + > +/* The single-thread optimization only works for libc itself, we need > + atomicity for libpthread in case of shared futexes. */ > +#if !IS_IN(libc) > +# define is_single_thread 0 > +#else > +# define is_single_thread SINGLE_THREAD_P > +#endif > + > +/* In the __lllc_as we simply return the value in %eax after the cmpxchg > + instruction. In case the operation succeded this value is zero. In case > + the operation failed, the cmpxchg instruction has loaded the current value > + of the memory work which is guaranteed to be nonzero. */ > +static inline int > +__attribute__ ((always_inline)) > +__lll_cas_lock (int *futex) > +{ > + int ret; > + asm volatile ("cmpxchgl %2, %1" > + : "=a" (ret), "=m" (*futex) > + : "r" (1), "m" (*futex), "0" (0) > + : "memory"); > + return ret; > +} > + > +#undef lll_trylock > +#define lll_trylock(lock) \ > + ({ \ > + int __ret; \ > + if (is_single_thread) \ > + __ret = __lll_cas_lock (&(lock)); \ > + else \ > + __ret = __lll_trylock (&(lock)); \ > + __ret; \ > + }) > + > +#undef lll_lock > +#define lll_lock(lock, private) \ > + ((void) \ > + ({ \ > + if (is_single_thread) \ > + __lll_cas_lock (&(lock)); \ > + else \ > + __lll_lock (&(lock), private); \ > + })) > + > +#undef lll_unlock > +#define lll_unlock(lock, private) \ > + ((void) \ > + ({ \ > + if (is_single_thread) \ > + (lock)--; \ > + else \ > + __lll_unlock (&(lock), private); \ > + })) > + > +extern int __lll_timedlock_elision (int *futex, short *adapt_count, > + const struct timespec *timeout, > + int private) attribute_hidden; > + > +#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ > + __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) > + > +extern int __lll_lock_elision (int *futex, short *adapt_count, int private) > + attribute_hidden; > + > +extern int __lll_unlock_elision (int *lock, int private) > + attribute_hidden; > + > +extern int __lll_trylock_elision (int *lock, short *adapt_count) > + attribute_hidden; > + > +#define lll_lock_elision(futex, adapt_count, private) \ > + __lll_lock_elision (&(futex), &(adapt_count), private) > +#define lll_unlock_elision(futex, adapt_count, private) \ > + __lll_unlock_elision (&(futex), private) > +#define lll_trylock_elision(futex, adapt_count) \ > + __lll_trylock_elision (&(futex), &(adapt_count)) > + > +#endif /* !__ASSEMBLER__ */ > + > +#endif /* lowlevellock.h */ > diff --git a/sysdeps/unix/sysv/linux/x86_64/cancellation.S b/sysdeps/unix/sysv/linux/x86_64/cancellation.S > index 7d169d9aca..bb4910764a 100644 > --- a/sysdeps/unix/sysv/linux/x86_64/cancellation.S > +++ b/sysdeps/unix/sysv/linux/x86_64/cancellation.S > @@ -19,7 +19,7 @@ > #include <sysdep.h> > #include <tcb-offsets.h> > #include <kernel-features.h> > -#include "lowlevellock.h" > +#include <lowlevellock-futex.h> > > #define PTHREAD_UNWIND JUMPTARGET(__pthread_unwind) > #if IS_IN (libpthread) > diff --git a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S > deleted file mode 100644 > index 32025277f3..0000000000 > --- a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S > +++ /dev/null > @@ -1,19 +0,0 @@ > -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <http://www.gnu.org/licenses/>. */ > - > -#include "lowlevellock.S" > diff --git a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c > deleted file mode 100644 > index f6875b8f89..0000000000 > --- a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c > +++ /dev/null > @@ -1 +0,0 @@ > -/* __lll_timedlock_wait is in lowlevellock.S. */ > diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S > deleted file mode 100644 > index 166dbcfd3b..0000000000 > --- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S > +++ /dev/null > @@ -1,348 +0,0 @@ > -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <http://www.gnu.org/licenses/>. */ > - > -#include <sysdep.h> > -#include <pthread-errnos.h> > -#include <kernel-features.h> > -#include <lowlevellock.h> > - > -#include <stap-probe.h> > - > - .text > - > -#define LOAD_PRIVATE_FUTEX_WAIT(reg) \ > - movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg > -#define LOAD_PRIVATE_FUTEX_WAKE(reg) \ > - movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg > -#define LOAD_FUTEX_WAIT(reg) \ > - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg > -#define LOAD_FUTEX_WAIT_ABS(reg) \ > - xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg > -#define LOAD_FUTEX_WAKE(reg) \ > - xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg > - > - > - .globl __lll_lock_wait_private > - .type __lll_lock_wait_private,@function > - .hidden __lll_lock_wait_private > - .align 16 > -__lll_lock_wait_private: > - cfi_startproc > - pushq %r10 > - cfi_adjust_cfa_offset(8) > - pushq %rdx > - cfi_adjust_cfa_offset(8) > - cfi_offset(%r10, -16) > - cfi_offset(%rdx, -24) > - xorq %r10, %r10 /* No timeout. */ > - movl $2, %edx > - LOAD_PRIVATE_FUTEX_WAIT (%esi) > - > - cmpl %edx, %eax /* NB: %edx == 2 */ > - jne 2f > - > -1: LIBC_PROBE (lll_lock_wait_private, 1, %rdi) > - movl $SYS_futex, %eax > - syscall > - > -2: movl %edx, %eax > - xchgl %eax, (%rdi) /* NB: lock is implied */ > - > - testl %eax, %eax > - jnz 1b > - > - popq %rdx > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%rdx) > - popq %r10 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r10) > - retq > - cfi_endproc > - .size __lll_lock_wait_private,.-__lll_lock_wait_private > - > -#if !IS_IN (libc) > - .globl __lll_lock_wait > - .type __lll_lock_wait,@function > - .hidden __lll_lock_wait > - .align 16 > -__lll_lock_wait: > - cfi_startproc > - pushq %r10 > - cfi_adjust_cfa_offset(8) > - pushq %rdx > - cfi_adjust_cfa_offset(8) > - cfi_offset(%r10, -16) > - cfi_offset(%rdx, -24) > - xorq %r10, %r10 /* No timeout. */ > - movl $2, %edx > - LOAD_FUTEX_WAIT (%esi) > - > - cmpl %edx, %eax /* NB: %edx == 2 */ > - jne 2f > - > -1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi) > - movl $SYS_futex, %eax > - syscall > - > -2: movl %edx, %eax > - xchgl %eax, (%rdi) /* NB: lock is implied */ > - > - testl %eax, %eax > - jnz 1b > - > - popq %rdx > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%rdx) > - popq %r10 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r10) > - retq > - cfi_endproc > - .size __lll_lock_wait,.-__lll_lock_wait > - > - /* %rdi: futex > - %rsi: flags > - %rdx: timeout > - %eax: futex value > - */ > - .globl __lll_timedlock_wait > - .type __lll_timedlock_wait,@function > - .hidden __lll_timedlock_wait > - .align 16 > -__lll_timedlock_wait: > - cfi_startproc > -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME > -# ifdef PIC > - cmpl $0, __have_futex_clock_realtime(%rip) > -# else > - cmpl $0, __have_futex_clock_realtime > -# endif > - je .Lreltmo > -# endif > - > - cmpq $0, (%rdx) > - js 5f > - > - pushq %r9 > - cfi_adjust_cfa_offset(8) > - cfi_rel_offset(%r9, 0) > - > - movq %rdx, %r10 > - movl $0xffffffff, %r9d > - LOAD_FUTEX_WAIT_ABS (%esi) > - > - movl $2, %edx > - cmpl %edx, %eax > - jne 2f > - > -1: movl $SYS_futex, %eax > - movl $2, %edx > - syscall > - > -2: xchgl %edx, (%rdi) /* NB: lock is implied */ > - > - testl %edx, %edx > - jz 3f > - > - cmpl $-ETIMEDOUT, %eax > - je 4f > - cmpl $-EINVAL, %eax > - jne 1b > -4: movl %eax, %edx > - negl %edx > - > -3: movl %edx, %eax > - popq %r9 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r9) > - retq > - > -5: movl $ETIMEDOUT, %eax > - retq > - > -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME > -.Lreltmo: > - /* Check for a valid timeout value. */ > - cmpq $1000000000, 8(%rdx) > - jae 3f > - > - pushq %r8 > - cfi_adjust_cfa_offset(8) > - pushq %r9 > - cfi_adjust_cfa_offset(8) > - pushq %r12 > - cfi_adjust_cfa_offset(8) > - pushq %r13 > - cfi_adjust_cfa_offset(8) > - pushq %r14 > - cfi_adjust_cfa_offset(8) > - cfi_offset(%r8, -16) > - cfi_offset(%r9, -24) > - cfi_offset(%r12, -32) > - cfi_offset(%r13, -40) > - cfi_offset(%r14, -48) > - pushq %rsi > - cfi_adjust_cfa_offset(8) > - > - /* Stack frame for the timespec and timeval structs. */ > - subq $24, %rsp > - cfi_adjust_cfa_offset(24) > - > - movq %rdi, %r12 > - movq %rdx, %r13 > - > - movl $2, %edx > - xchgl %edx, (%r12) > - > - testl %edx, %edx > - je 6f > - > -1: > - /* Get current time. */ > - movq %rsp, %rdi > - xorl %esi, %esi > - /* This call works because we directly jump to a system call entry > - which preserves all the registers. */ > - call JUMPTARGET(__gettimeofday) > - > - /* Compute relative timeout. */ > - movq 8(%rsp), %rax > - movl $1000, %edi > - mul %rdi /* Milli seconds to nano seconds. */ > - movq (%r13), %rdi > - movq 8(%r13), %rsi > - subq (%rsp), %rdi > - subq %rax, %rsi > - jns 4f > - addq $1000000000, %rsi > - decq %rdi > -4: testq %rdi, %rdi > - js 2f /* Time is already up. */ > - > - /* Store relative timeout. */ > - movq %rdi, (%rsp) > - movq %rsi, 8(%rsp) > - > - /* Futex call. */ > - movl $2, %edx > - movl $1, %eax > - movq %rsp, %r10 > - movl 24(%rsp), %esi > - LOAD_FUTEX_WAIT (%esi) > - movq %r12, %rdi > - movl $SYS_futex, %eax > - syscall > - > - /* NB: %edx == 2 */ > - xchgl %edx, (%r12) > - > - testl %edx, %edx > - je 6f > - > - cmpl $-ETIMEDOUT, %eax > - jne 1b > -2: movl $ETIMEDOUT, %edx > - > -6: addq $32, %rsp > - cfi_adjust_cfa_offset(-32) > - popq %r14 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r14) > - popq %r13 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r13) > - popq %r12 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r12) > - popq %r9 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r9) > - popq %r8 > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%r8) > - movl %edx, %eax > - retq > - > -3: movl $EINVAL, %eax > - retq > -# endif > - cfi_endproc > - .size __lll_timedlock_wait,.-__lll_timedlock_wait > -#endif > - > - > - .globl __lll_unlock_wake_private > - .type __lll_unlock_wake_private,@function > - .hidden __lll_unlock_wake_private > - .align 16 > -__lll_unlock_wake_private: > - cfi_startproc > - pushq %rsi > - cfi_adjust_cfa_offset(8) > - pushq %rdx > - cfi_adjust_cfa_offset(8) > - cfi_offset(%rsi, -16) > - cfi_offset(%rdx, -24) > - > - movl $0, (%rdi) > - LOAD_PRIVATE_FUTEX_WAKE (%esi) > - movl $1, %edx /* Wake one thread. */ > - movl $SYS_futex, %eax > - syscall > - > - popq %rdx > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%rdx) > - popq %rsi > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%rsi) > - retq > - cfi_endproc > - .size __lll_unlock_wake_private,.-__lll_unlock_wake_private > - > -#if !IS_IN (libc) > - .globl __lll_unlock_wake > - .type __lll_unlock_wake,@function > - .hidden __lll_unlock_wake > - .align 16 > -__lll_unlock_wake: > - cfi_startproc > - pushq %rsi > - cfi_adjust_cfa_offset(8) > - pushq %rdx > - cfi_adjust_cfa_offset(8) > - cfi_offset(%rsi, -16) > - cfi_offset(%rdx, -24) > - > - movl $0, (%rdi) > - LOAD_FUTEX_WAKE (%esi) > - movl $1, %edx /* Wake one thread. */ > - movl $SYS_futex, %eax > - syscall > - > - popq %rdx > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%rdx) > - popq %rsi > - cfi_adjust_cfa_offset(-8) > - cfi_restore(%rsi) > - retq > - cfi_endproc > - .size __lll_unlock_wake,.-__lll_unlock_wake > -#endif > diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h > deleted file mode 100644 > index 8cbc1caa5b..0000000000 > --- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h > +++ /dev/null > @@ -1,243 +0,0 @@ > -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <http://www.gnu.org/licenses/>. */ > - > -#ifndef _LOWLEVELLOCK_H > -#define _LOWLEVELLOCK_H 1 > - > -#ifndef __ASSEMBLER__ > -# include <time.h> > -# include <sys/param.h> > -# include <bits/pthreadtypes.h> > -# include <kernel-features.h> > - > -# ifndef LOCK_INSTR > -# ifdef UP > -# define LOCK_INSTR /* nothing */ > -# else > -# define LOCK_INSTR "lock;" > -# endif > -# endif > -#else > -# ifndef LOCK > -# ifdef UP > -# define LOCK > -# else > -# define LOCK lock > -# endif > -# endif > -#endif > - > -#include <lowlevellock-futex.h> > - > -/* XXX Remove when no assembler code uses futexes anymore. */ > -#define SYS_futex __NR_futex > - > -#ifndef __ASSEMBLER__ > - > -/* Initializer for lock. */ > -#define LLL_LOCK_INITIALIZER (0) > -#define LLL_LOCK_INITIALIZER_LOCKED (1) > -#define LLL_LOCK_INITIALIZER_WAITERS (2) > - > - > -/* NB: in the lll_trylock macro we simply return the value in %eax > - after the cmpxchg instruction. In case the operation succeded this > - value is zero. In case the operation failed, the cmpxchg instruction > - has loaded the current value of the memory work which is guaranteed > - to be nonzero. */ > -#if !IS_IN (libc) || defined UP > -# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" > -#else > -# define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ > - "je 0f\n\t" \ > - "lock; cmpxchgl %2, %1\n\t" \ > - "jmp 1f\n\t" \ > - "0:\tcmpxchgl %2, %1\n\t" \ > - "1:" > -#endif > - > -#define lll_trylock(futex) \ > - ({ int ret; \ > - __asm __volatile (__lll_trylock_asm \ > - : "=a" (ret), "=m" (futex) \ > - : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ > - "0" (LLL_LOCK_INITIALIZER) \ > - : "memory"); \ > - ret; }) > - > -#define lll_cond_trylock(futex) \ > - ({ int ret; \ > - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ > - : "=a" (ret), "=m" (futex) \ > - : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ > - "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ > - : "memory"); \ > - ret; }) > - > -#if !IS_IN (libc) || defined UP > -# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \ > - "jz 24f\n\t" > -#else > -# define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ > - "je 0f\n\t" \ > - "lock; cmpxchgl %4, %2\n\t" \ > - "jnz 1f\n\t" \ > - "jmp 24f\n" \ > - "0:\tcmpxchgl %4, %2\n\t" \ > - "jz 24f\n\t" > -#endif > - > -#define lll_lock(futex, private) \ > - (void) \ > - ({ int ignore1, ignore2, ignore3; \ > - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ > - __asm __volatile (__lll_lock_asm_start \ > - "1:\tlea %2, %%" RDI_LP "\n" \ > - "2:\tsub $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset 128\n" \ > - "3:\tcallq __lll_lock_wait_private\n" \ > - "4:\tadd $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset -128\n" \ > - "24:" \ > - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ > - "=a" (ignore3) \ > - : "0" (1), "m" (futex), "3" (0) \ > - : "cx", "r11", "cc", "memory"); \ > - else \ > - __asm __volatile (__lll_lock_asm_start \ > - "1:\tlea %2, %%" RDI_LP "\n" \ > - "2:\tsub $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset 128\n" \ > - "3:\tcallq __lll_lock_wait\n" \ > - "4:\tadd $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset -128\n" \ > - "24:" \ > - : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ > - "=a" (ignore3) \ > - : "1" (1), "m" (futex), "3" (0), "0" (private) \ > - : "cx", "r11", "cc", "memory"); \ > - }) \ > - > -#define lll_cond_lock(futex, private) \ > - (void) \ > - ({ int ignore1, ignore2, ignore3; \ > - __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ > - "jz 24f\n" \ > - "1:\tlea %2, %%" RDI_LP "\n" \ > - "2:\tsub $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset 128\n" \ > - "3:\tcallq __lll_lock_wait\n" \ > - "4:\tadd $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset -128\n" \ > - "24:" \ > - : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ > - "=a" (ignore3) \ > - : "1" (2), "m" (futex), "3" (0), "0" (private) \ > - : "cx", "r11", "cc", "memory"); \ > - }) > - > -#define lll_timedlock(futex, timeout, private) \ > - ({ int result, ignore1, ignore2, ignore3; \ > - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ > - "jz 24f\n" \ > - "1:\tlea %4, %%" RDI_LP "\n" \ > - "0:\tmov %8, %%" RDX_LP "\n" \ > - "2:\tsub $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset 128\n" \ > - "3:\tcallq __lll_timedlock_wait\n" \ > - "4:\tadd $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset -128\n" \ > - "24:" \ > - : "=a" (result), "=D" (ignore1), "=S" (ignore2), \ > - "=&d" (ignore3), "=m" (futex) \ > - : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ > - "2" (private) \ > - : "memory", "cx", "cc", "r10", "r11"); \ > - result; }) > - > -extern int __lll_timedlock_elision (int *futex, short *adapt_count, > - const struct timespec *timeout, > - int private) attribute_hidden; > - > -#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ > - __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) > - > -#if !IS_IN (libc) || defined UP > -# define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \ > - "je 24f\n\t" > -#else > -# define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ > - "je 0f\n\t" \ > - "lock; decl %0\n\t" \ > - "jne 1f\n\t" \ > - "jmp 24f\n\t" \ > - "0:\tdecl %0\n\t" \ > - "je 24f\n\t" > -#endif > - > -#define lll_unlock(futex, private) \ > - (void) \ > - ({ int ignore; \ > - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ > - __asm __volatile (__lll_unlock_asm_start \ > - "1:\tlea %0, %%" RDI_LP "\n" \ > - "2:\tsub $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset 128\n" \ > - "3:\tcallq __lll_unlock_wake_private\n" \ > - "4:\tadd $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset -128\n" \ > - "24:" \ > - : "=m" (futex), "=&D" (ignore) \ > - : "m" (futex) \ > - : "ax", "cx", "r11", "cc", "memory"); \ > - else \ > - __asm __volatile (__lll_unlock_asm_start \ > - "1:\tlea %0, %%" RDI_LP "\n" \ > - "2:\tsub $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset 128\n" \ > - "3:\tcallq __lll_unlock_wake\n" \ > - "4:\tadd $128, %%" RSP_LP "\n" \ > - ".cfi_adjust_cfa_offset -128\n" \ > - "24:" \ > - : "=m" (futex), "=&D" (ignore) \ > - : "m" (futex), "S" (private) \ > - : "ax", "cx", "r11", "cc", "memory"); \ > - }) > - > -#define lll_islocked(futex) \ > - (futex != LLL_LOCK_INITIALIZER) > - > -extern int __lll_lock_elision (int *futex, short *adapt_count, int private) > - attribute_hidden; > - > -extern int __lll_unlock_elision (int *lock, int private) > - attribute_hidden; > - > -extern int __lll_trylock_elision (int *lock, short *adapt_count) > - attribute_hidden; > - > -#define lll_lock_elision(futex, adapt_count, private) \ > - __lll_lock_elision (&(futex), &(adapt_count), private) > -#define lll_unlock_elision(futex, adapt_count, private) \ > - __lll_unlock_elision (&(futex), private) > -#define lll_trylock_elision(futex, adapt_count) \ > - __lll_trylock_elision (&(futex), &(adapt_count)) > - > -#endif /* !__ASSEMBLER__ */ > - > -#endif /* lowlevellock.h */ >
On Fri, Feb 22, 2019 at 11:27 AM Adhemerval Zanella <adhemerval.zanella@linaro.org> wrote: > > This patch removes the arch-specific x86 assembly implementation for > low level locking and consolidate both 64 bits and 32 bits in a > single implementation. > > Different than other architectures, x86 lll_trylock, lll_lock, and > lll_unlock implements a single-thread optimization to avoid atomic > operation, using cmpxchgl instead. This patch implements by using > the new single-thread.h definitions in a generic way, although using > the previous semantic. > > The lll_cond_trylock, lll_cond_lock, and lll_timedlock just use > atomic operations plus calls to lll_lock_wait*. > > For __lll_lock_wait_private and __lll_lock_wait the generic implemtation > there is no indication that assembly implementation is required > performance-wise. > > Checked on x86_64-linux-gnu and i686-linux-gnu. > > * sysdeps/nptl/lowlevellock.h (__lll_trylock): New macro. > (lll_trylock): Call __lll_trylock. > * sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S: Remove file. > * sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c: Likewise. > * sysdeps/unix/sysv/linux/i386/lowlevellock.S: Likewise. > * sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise. > * sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Likewise. > * sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c: Likewise. > * sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Likewise. > * sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Likewise. > * sysdeps/unix/sysv/linux/x86/lowlevellock.h: New file. > * sysdeps/unix/sysv/linux/x86_64/cancellation.S: Include > lowlevellock-futex.h. > --- > sysdeps/nptl/lowlevellock.h | 4 +- > .../unix/sysv/linux/i386/libc-lowlevellock.S | 19 - > .../unix/sysv/linux/i386/lll_timedlock_wait.c | 1 - > sysdeps/unix/sysv/linux/i386/lowlevellock.S | 368 ------------------ > sysdeps/unix/sysv/linux/i386/lowlevellock.h | 240 ------------ > sysdeps/unix/sysv/linux/x86/lowlevellock.h | 110 ++++++ > sysdeps/unix/sysv/linux/x86_64/cancellation.S | 2 +- > .../sysv/linux/x86_64/libc-lowlevellock.S | 19 - > .../sysv/linux/x86_64/lll_timedlock_wait.c | 1 - > sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 348 ----------------- > sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 243 ------------ > 11 files changed, 114 insertions(+), 1241 deletions(-) > delete mode 100644 sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c > delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/i386/lowlevellock.h > create mode 100644 sysdeps/unix/sysv/linux/x86/lowlevellock.h > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.S > delete mode 100644 sysdeps/unix/sysv/linux/x86_64/lowlevellock.h > LGTM. Thanks. -- H.J.
diff --git a/sysdeps/nptl/lowlevellock.h b/sysdeps/nptl/lowlevellock.h index 6f017afdd5..e905829ee4 100644 --- a/sysdeps/nptl/lowlevellock.h +++ b/sysdeps/nptl/lowlevellock.h @@ -63,8 +63,10 @@ /* If LOCK is 0 (not acquired), set to 1 (acquired with no waiters) and return 0. Otherwise leave lock unchanged and return non-zero to indicate that the lock was not acquired. */ +#define __lll_trylock(lock) \ + __glibc_unlikely (atomic_compare_and_exchange_bool_acq ((lock), 1, 0)) #define lll_trylock(lock) \ - __glibc_unlikely (atomic_compare_and_exchange_bool_acq (&(lock), 1, 0)) + __lll_trylock (&(lock)) /* If LOCK is 0 (not acquired), set to 2 (acquired, possibly with waiters) and return 0. Otherwise leave lock unchanged and return non-zero to indicate diff --git a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S deleted file mode 100644 index 32025277f3..0000000000 --- a/sysdeps/unix/sysv/linux/i386/libc-lowlevellock.S +++ /dev/null @@ -1,19 +0,0 @@ -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include "lowlevellock.S" diff --git a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c deleted file mode 100644 index f6875b8f89..0000000000 --- a/sysdeps/unix/sysv/linux/i386/lll_timedlock_wait.c +++ /dev/null @@ -1 +0,0 @@ -/* __lll_timedlock_wait is in lowlevellock.S. */ diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.S b/sysdeps/unix/sysv/linux/i386/lowlevellock.S deleted file mode 100644 index 83191a3899..0000000000 --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.S +++ /dev/null @@ -1,368 +0,0 @@ -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <pthread-errnos.h> -#include <kernel-features.h> -#include <lowlevellock.h> - -#include <stap-probe.h> - - .text - -#define LOAD_PRIVATE_FUTEX_WAIT(reg) \ - movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg -#define LOAD_PRIVATE_FUTEX_WAKE(reg) \ - movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg -#define LOAD_FUTEX_WAIT(reg) \ - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg -#define LOAD_FUTEX_WAIT_ABS(reg) \ - xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg -#define LOAD_FUTEX_WAKE(reg) \ - xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg - - .globl __lll_lock_wait_private - .type __lll_lock_wait_private,@function - .hidden __lll_lock_wait_private - .align 16 -__lll_lock_wait_private: - cfi_startproc - pushl %edx - cfi_adjust_cfa_offset(4) - pushl %ebx - cfi_adjust_cfa_offset(4) - pushl %esi - cfi_adjust_cfa_offset(4) - cfi_offset(%edx, -8) - cfi_offset(%ebx, -12) - cfi_offset(%esi, -16) - - movl $2, %edx - movl %ecx, %ebx - xorl %esi, %esi /* No timeout. */ - LOAD_PRIVATE_FUTEX_WAIT (%ecx) - - cmpl %edx, %eax /* NB: %edx == 2 */ - jne 2f - -1: LIBC_PROBE (lll_lock_wait_private, 1, %ebx) - movl $SYS_futex, %eax - ENTER_KERNEL - -2: movl %edx, %eax - xchgl %eax, (%ebx) /* NB: lock is implied */ - - testl %eax, %eax - jnz 1b - - popl %esi - cfi_adjust_cfa_offset(-4) - cfi_restore(%esi) - popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - popl %edx - cfi_adjust_cfa_offset(-4) - cfi_restore(%edx) - ret - cfi_endproc - .size __lll_lock_wait_private,.-__lll_lock_wait_private - -#if !IS_IN (libc) - .globl __lll_lock_wait - .type __lll_lock_wait,@function - .hidden __lll_lock_wait - .align 16 -__lll_lock_wait: - cfi_startproc - pushl %edx - cfi_adjust_cfa_offset(4) - pushl %ebx - cfi_adjust_cfa_offset(4) - pushl %esi - cfi_adjust_cfa_offset(4) - cfi_offset(%edx, -8) - cfi_offset(%ebx, -12) - cfi_offset(%esi, -16) - - movl %edx, %ebx - movl $2, %edx - xorl %esi, %esi /* No timeout. */ - LOAD_FUTEX_WAIT (%ecx) - - cmpl %edx, %eax /* NB: %edx == 2 */ - jne 2f - -1: movl $SYS_futex, %eax - ENTER_KERNEL - -2: movl %edx, %eax - xchgl %eax, (%ebx) /* NB: lock is implied */ - - testl %eax, %eax - jnz 1b - - popl %esi - cfi_adjust_cfa_offset(-4) - cfi_restore(%esi) - popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - popl %edx - cfi_adjust_cfa_offset(-4) - cfi_restore(%edx) - ret - cfi_endproc - .size __lll_lock_wait,.-__lll_lock_wait - - /* %ecx: futex - %esi: flags - %edx: timeout - %eax: futex value - */ - .globl __lll_timedlock_wait - .type __lll_timedlock_wait,@function - .hidden __lll_timedlock_wait - .align 16 -__lll_timedlock_wait: - cfi_startproc - pushl %ebp - cfi_adjust_cfa_offset(4) - cfi_rel_offset(%ebp, 0) - pushl %ebx - cfi_adjust_cfa_offset(4) - cfi_rel_offset(%ebx, 0) - -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME -# ifdef PIC - LOAD_PIC_REG (bx) - cmpl $0, __have_futex_clock_realtime@GOTOFF(%ebx) -# else - cmpl $0, __have_futex_clock_realtime -# endif - je .Lreltmo -# endif - - cmpl $0, (%edx) - js 8f - - movl %ecx, %ebx - movl %esi, %ecx - movl %edx, %esi - movl $0xffffffff, %ebp - LOAD_FUTEX_WAIT_ABS (%ecx) - - movl $2, %edx - cmpl %edx, %eax - jne 2f - -1: movl $SYS_futex, %eax - movl $2, %edx - ENTER_KERNEL - -2: xchgl %edx, (%ebx) /* NB: lock is implied */ - - testl %edx, %edx - jz 3f - - cmpl $-ETIMEDOUT, %eax - je 4f - cmpl $-EINVAL, %eax - jne 1b -4: movl %eax, %edx - negl %edx - -3: movl %edx, %eax -7: popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - popl %ebp - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebp) - ret - -8: movl $ETIMEDOUT, %eax - jmp 7b - -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME -.Lreltmo: - /* Check for a valid timeout value. */ - cmpl $1000000000, 4(%edx) - jae 3f - - pushl %esi - cfi_adjust_cfa_offset(4) - cfi_rel_offset(%esi, 0) - pushl %edi - cfi_adjust_cfa_offset(4) - cfi_rel_offset(%edi, 0) - - /* Stack frame for the timespec and timeval structs. */ - subl $8, %esp - cfi_adjust_cfa_offset(8) - - movl %ecx, %ebp - movl %edx, %edi - - movl $2, %edx - xchgl %edx, (%ebp) - - test %edx, %edx - je 6f - -1: - /* Get current time. */ - movl %esp, %ebx - xorl %ecx, %ecx - movl $__NR_gettimeofday, %eax - ENTER_KERNEL - - /* Compute relative timeout. */ - movl 4(%esp), %eax - movl $1000, %edx - mul %edx /* Milli seconds to nano seconds. */ - movl (%edi), %ecx - movl 4(%edi), %edx - subl (%esp), %ecx - subl %eax, %edx - jns 4f - addl $1000000000, %edx - subl $1, %ecx -4: testl %ecx, %ecx - js 2f /* Time is already up. */ - - /* Store relative timeout. */ - movl %ecx, (%esp) - movl %edx, 4(%esp) - - /* Futex call. */ - movl %ebp, %ebx - movl $2, %edx - movl %esp, %esi - movl 16(%esp), %ecx - LOAD_FUTEX_WAIT (%ecx) - movl $SYS_futex, %eax - ENTER_KERNEL - - /* NB: %edx == 2 */ - xchgl %edx, (%ebp) - - testl %edx, %edx - je 6f - - cmpl $-ETIMEDOUT, %eax - jne 1b -2: movl $ETIMEDOUT, %edx - -6: addl $8, %esp - cfi_adjust_cfa_offset(-8) - popl %edi - cfi_adjust_cfa_offset(-4) - cfi_restore(%edi) - popl %esi - cfi_adjust_cfa_offset(-4) - cfi_restore(%esi) -7: popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - popl %ebp - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebp) - movl %edx, %eax - ret - -3: movl $EINVAL, %edx - jmp 7b -# endif - cfi_endproc - .size __lll_timedlock_wait,.-__lll_timedlock_wait -#endif - - .globl __lll_unlock_wake_private - .type __lll_unlock_wake_private,@function - .hidden __lll_unlock_wake_private - .align 16 -__lll_unlock_wake_private: - cfi_startproc - pushl %ebx - cfi_adjust_cfa_offset(4) - pushl %ecx - cfi_adjust_cfa_offset(4) - pushl %edx - cfi_adjust_cfa_offset(4) - cfi_offset(%ebx, -8) - cfi_offset(%ecx, -12) - cfi_offset(%edx, -16) - - movl %eax, %ebx - movl $0, (%eax) - LOAD_PRIVATE_FUTEX_WAKE (%ecx) - movl $1, %edx /* Wake one thread. */ - movl $SYS_futex, %eax - ENTER_KERNEL - - popl %edx - cfi_adjust_cfa_offset(-4) - cfi_restore(%edx) - popl %ecx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ecx) - popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - ret - cfi_endproc - .size __lll_unlock_wake_private,.-__lll_unlock_wake_private - -#if !IS_IN (libc) - .globl __lll_unlock_wake - .type __lll_unlock_wake,@function - .hidden __lll_unlock_wake - .align 16 -__lll_unlock_wake: - cfi_startproc - pushl %ebx - cfi_adjust_cfa_offset(4) - pushl %ecx - cfi_adjust_cfa_offset(4) - pushl %edx - cfi_adjust_cfa_offset(4) - cfi_offset(%ebx, -8) - cfi_offset(%ecx, -12) - cfi_offset(%edx, -16) - - movl %eax, %ebx - movl $0, (%eax) - LOAD_FUTEX_WAKE (%ecx) - movl $1, %edx /* Wake one thread. */ - movl $SYS_futex, %eax - ENTER_KERNEL - - popl %edx - cfi_adjust_cfa_offset(-4) - cfi_restore(%edx) - popl %ecx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ecx) - popl %ebx - cfi_adjust_cfa_offset(-4) - cfi_restore(%ebx) - ret - cfi_endproc - .size __lll_unlock_wake,.-__lll_unlock_wake -#endif diff --git a/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/sysdeps/unix/sysv/linux/i386/lowlevellock.h deleted file mode 100644 index 94dccc4ce7..0000000000 --- a/sysdeps/unix/sysv/linux/i386/lowlevellock.h +++ /dev/null @@ -1,240 +0,0 @@ -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#ifndef _LOWLEVELLOCK_H -#define _LOWLEVELLOCK_H 1 - -#ifndef __ASSEMBLER__ -# include <time.h> -# include <sys/param.h> -# include <bits/pthreadtypes.h> -# include <kernel-features.h> -/* <tcb-offsets.h> is generated from tcb-offsets.sym to define offsets - and sizes of types in <tls.h> as well as <pthread.h> which includes - <lowlevellock.h> via nptl/descr.h. Don't include <tcb-offsets.h> - when generating <tcb-offsets.h> to avoid circular dependency which - may lead to build hang on a many-core machine. */ -# ifndef GEN_AS_CONST_HEADERS -# include <tcb-offsets.h> -# endif - -# ifndef LOCK_INSTR -# ifdef UP -# define LOCK_INSTR /* nothing */ -# else -# define LOCK_INSTR "lock;" -# endif -# endif -#else -# ifndef LOCK -# ifdef UP -# define LOCK -# else -# define LOCK lock -# endif -# endif -#endif - -#include <lowlevellock-futex.h> - -/* XXX Remove when no assembler code uses futexes anymore. */ -#define SYS_futex __NR_futex - -#ifndef __ASSEMBLER__ - -/* Initializer for compatibility lock. */ -#define LLL_LOCK_INITIALIZER (0) -#define LLL_LOCK_INITIALIZER_LOCKED (1) -#define LLL_LOCK_INITIALIZER_WAITERS (2) - - -/* NB: in the lll_trylock macro we simply return the value in %eax - after the cmpxchg instruction. In case the operation succeded this - value is zero. In case the operation failed, the cmpxchg instruction - has loaded the current value of the memory work which is guaranteed - to be nonzero. */ -#if !IS_IN (libc) || defined UP -# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" -#else -# define __lll_trylock_asm "cmpl $0, %%gs:%P5\n\t" \ - "je 0f\n\t" \ - "lock\n" \ - "0:\tcmpxchgl %2, %1" -#endif - -#define lll_trylock(futex) \ - ({ int ret; \ - __asm __volatile (__lll_trylock_asm \ - : "=a" (ret), "=m" (futex) \ - : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ - "0" (LLL_LOCK_INITIALIZER), \ - "i" (MULTIPLE_THREADS_OFFSET) \ - : "memory"); \ - ret; }) - - -#define lll_cond_trylock(futex) \ - ({ int ret; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ - : "=a" (ret), "=m" (futex) \ - : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ - "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ - : "memory"); \ - ret; }) - -#if !IS_IN (libc) || defined UP -# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %1, %2\n\t" -#else -# define __lll_lock_asm_start "cmpl $0, %%gs:%P6\n\t" \ - "je 0f\n\t" \ - "lock\n" \ - "0:\tcmpxchgl %1, %2\n\t" -#endif - -#define lll_lock(futex, private) \ - (void) \ - ({ int ignore1, ignore2; \ - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ - __asm __volatile (__lll_lock_asm_start \ - "jz 18f\n\t" \ - "1:\tleal %2, %%ecx\n" \ - "2:\tcall __lll_lock_wait_private\n" \ - "18:" \ - : "=a" (ignore1), "=c" (ignore2), "=m" (futex) \ - : "0" (0), "1" (1), "m" (futex), \ - "i" (MULTIPLE_THREADS_OFFSET) \ - : "memory"); \ - else \ - { \ - int ignore3; \ - __asm __volatile (__lll_lock_asm_start \ - "jz 18f\n\t" \ - "1:\tleal %2, %%edx\n" \ - "0:\tmovl %8, %%ecx\n" \ - "2:\tcall __lll_lock_wait\n" \ - "18:" \ - : "=a" (ignore1), "=c" (ignore2), \ - "=m" (futex), "=&d" (ignore3) \ - : "1" (1), "m" (futex), \ - "i" (MULTIPLE_THREADS_OFFSET), "0" (0), \ - "g" ((int) (private)) \ - : "memory"); \ - } \ - }) - - -/* Special version of lll_lock which causes the unlock function to - always wakeup waiters. */ -#define lll_cond_lock(futex, private) \ - (void) \ - ({ int ignore1, ignore2, ignore3; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t" \ - "jz 18f\n\t" \ - "1:\tleal %2, %%edx\n" \ - "0:\tmovl %7, %%ecx\n" \ - "2:\tcall __lll_lock_wait\n" \ - "18:" \ - : "=a" (ignore1), "=c" (ignore2), "=m" (futex), \ - "=&d" (ignore3) \ - : "0" (0), "1" (2), "m" (futex), "g" ((int) (private))\ - : "memory"); \ - }) - - -#define lll_timedlock(futex, timeout, private) \ - ({ int result, ignore1, ignore2, ignore3; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t" \ - "jz 18f\n\t" \ - "1:\tleal %3, %%ecx\n" \ - "0:\tmovl %8, %%edx\n" \ - "2:\tcall __lll_timedlock_wait\n" \ - "18:" \ - : "=a" (result), "=c" (ignore1), "=&d" (ignore2), \ - "=m" (futex), "=S" (ignore3) \ - : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ - "4" ((int) (private)) \ - : "memory"); \ - result; }) - -extern int __lll_timedlock_elision (int *futex, short *adapt_count, - const struct timespec *timeout, - int private) attribute_hidden; - -#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ - __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) - -#if !IS_IN (libc) || defined UP -# define __lll_unlock_asm LOCK_INSTR "subl $1, %0\n\t" -#else -# define __lll_unlock_asm "cmpl $0, %%gs:%P3\n\t" \ - "je 0f\n\t" \ - "lock\n" \ - "0:\tsubl $1,%0\n\t" -#endif - -#define lll_unlock(futex, private) \ - (void) \ - ({ int ignore; \ - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ - __asm __volatile (__lll_unlock_asm \ - "je 18f\n\t" \ - "1:\tleal %0, %%eax\n" \ - "2:\tcall __lll_unlock_wake_private\n" \ - "18:" \ - : "=m" (futex), "=&a" (ignore) \ - : "m" (futex), "i" (MULTIPLE_THREADS_OFFSET) \ - : "memory"); \ - else \ - { \ - int ignore2; \ - __asm __volatile (__lll_unlock_asm \ - "je 18f\n\t" \ - "1:\tleal %0, %%eax\n" \ - "0:\tmovl %5, %%ecx\n" \ - "2:\tcall __lll_unlock_wake\n" \ - "18:" \ - : "=m" (futex), "=&a" (ignore), "=&c" (ignore2) \ - : "i" (MULTIPLE_THREADS_OFFSET), "m" (futex), \ - "g" ((int) (private)) \ - : "memory"); \ - } \ - }) - - -#define lll_islocked(futex) \ - (futex != LLL_LOCK_INITIALIZER) - -extern int __lll_lock_elision (int *futex, short *adapt_count, int private) - attribute_hidden; - -extern int __lll_unlock_elision(int *lock, int private) - attribute_hidden; - -extern int __lll_trylock_elision(int *lock, short *adapt_count) - attribute_hidden; - -#define lll_lock_elision(futex, adapt_count, private) \ - __lll_lock_elision (&(futex), &(adapt_count), private) -#define lll_unlock_elision(futex, adapt_count, private) \ - __lll_unlock_elision (&(futex), private) -#define lll_trylock_elision(futex, adapt_count) \ - __lll_trylock_elision(&(futex), &(adapt_count)) - -#endif /* !__ASSEMBLER__ */ - -#endif /* lowlevellock.h */ diff --git a/sysdeps/unix/sysv/linux/x86/lowlevellock.h b/sysdeps/unix/sysv/linux/x86/lowlevellock.h new file mode 100644 index 0000000000..2652a00736 --- /dev/null +++ b/sysdeps/unix/sysv/linux/x86/lowlevellock.h @@ -0,0 +1,110 @@ +/* Low-level lock implementation, x86 version. + Copyright (C) 2019 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#ifndef _X86_64_LOWLEVELLOCK_H +#define _X86_64_LOWLEVELLOCK_H 1 + +#ifndef __ASSEMBLER__ +#include <sysdeps/nptl/lowlevellock.h> +#include <single-thread.h> + +/* The lll_trylock, lll_lock, and lll_unlock implements a single-thread + optimization using the cmpxchgl instruction. It checks if the process + is single thread and avoid a more expensive atomic instruction. */ + +/* The single-thread optimization only works for libc itself, we need + atomicity for libpthread in case of shared futexes. */ +#if !IS_IN(libc) +# define is_single_thread 0 +#else +# define is_single_thread SINGLE_THREAD_P +#endif + +/* In the __lllc_as we simply return the value in %eax after the cmpxchg + instruction. In case the operation succeded this value is zero. In case + the operation failed, the cmpxchg instruction has loaded the current value + of the memory work which is guaranteed to be nonzero. */ +static inline int +__attribute__ ((always_inline)) +__lll_cas_lock (int *futex) +{ + int ret; + asm volatile ("cmpxchgl %2, %1" + : "=a" (ret), "=m" (*futex) + : "r" (1), "m" (*futex), "0" (0) + : "memory"); + return ret; +} + +#undef lll_trylock +#define lll_trylock(lock) \ + ({ \ + int __ret; \ + if (is_single_thread) \ + __ret = __lll_cas_lock (&(lock)); \ + else \ + __ret = __lll_trylock (&(lock)); \ + __ret; \ + }) + +#undef lll_lock +#define lll_lock(lock, private) \ + ((void) \ + ({ \ + if (is_single_thread) \ + __lll_cas_lock (&(lock)); \ + else \ + __lll_lock (&(lock), private); \ + })) + +#undef lll_unlock +#define lll_unlock(lock, private) \ + ((void) \ + ({ \ + if (is_single_thread) \ + (lock)--; \ + else \ + __lll_unlock (&(lock), private); \ + })) + +extern int __lll_timedlock_elision (int *futex, short *adapt_count, + const struct timespec *timeout, + int private) attribute_hidden; + +#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ + __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) + +extern int __lll_lock_elision (int *futex, short *adapt_count, int private) + attribute_hidden; + +extern int __lll_unlock_elision (int *lock, int private) + attribute_hidden; + +extern int __lll_trylock_elision (int *lock, short *adapt_count) + attribute_hidden; + +#define lll_lock_elision(futex, adapt_count, private) \ + __lll_lock_elision (&(futex), &(adapt_count), private) +#define lll_unlock_elision(futex, adapt_count, private) \ + __lll_unlock_elision (&(futex), private) +#define lll_trylock_elision(futex, adapt_count) \ + __lll_trylock_elision (&(futex), &(adapt_count)) + +#endif /* !__ASSEMBLER__ */ + +#endif /* lowlevellock.h */ diff --git a/sysdeps/unix/sysv/linux/x86_64/cancellation.S b/sysdeps/unix/sysv/linux/x86_64/cancellation.S index 7d169d9aca..bb4910764a 100644 --- a/sysdeps/unix/sysv/linux/x86_64/cancellation.S +++ b/sysdeps/unix/sysv/linux/x86_64/cancellation.S @@ -19,7 +19,7 @@ #include <sysdep.h> #include <tcb-offsets.h> #include <kernel-features.h> -#include "lowlevellock.h" +#include <lowlevellock-futex.h> #define PTHREAD_UNWIND JUMPTARGET(__pthread_unwind) #if IS_IN (libpthread) diff --git a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S deleted file mode 100644 index 32025277f3..0000000000 --- a/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S +++ /dev/null @@ -1,19 +0,0 @@ -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include "lowlevellock.S" diff --git a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c b/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c deleted file mode 100644 index f6875b8f89..0000000000 --- a/sysdeps/unix/sysv/linux/x86_64/lll_timedlock_wait.c +++ /dev/null @@ -1 +0,0 @@ -/* __lll_timedlock_wait is in lowlevellock.S. */ diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S deleted file mode 100644 index 166dbcfd3b..0000000000 --- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ /dev/null @@ -1,348 +0,0 @@ -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <pthread-errnos.h> -#include <kernel-features.h> -#include <lowlevellock.h> - -#include <stap-probe.h> - - .text - -#define LOAD_PRIVATE_FUTEX_WAIT(reg) \ - movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg -#define LOAD_PRIVATE_FUTEX_WAKE(reg) \ - movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg -#define LOAD_FUTEX_WAIT(reg) \ - xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg -#define LOAD_FUTEX_WAIT_ABS(reg) \ - xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg -#define LOAD_FUTEX_WAKE(reg) \ - xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg - - - .globl __lll_lock_wait_private - .type __lll_lock_wait_private,@function - .hidden __lll_lock_wait_private - .align 16 -__lll_lock_wait_private: - cfi_startproc - pushq %r10 - cfi_adjust_cfa_offset(8) - pushq %rdx - cfi_adjust_cfa_offset(8) - cfi_offset(%r10, -16) - cfi_offset(%rdx, -24) - xorq %r10, %r10 /* No timeout. */ - movl $2, %edx - LOAD_PRIVATE_FUTEX_WAIT (%esi) - - cmpl %edx, %eax /* NB: %edx == 2 */ - jne 2f - -1: LIBC_PROBE (lll_lock_wait_private, 1, %rdi) - movl $SYS_futex, %eax - syscall - -2: movl %edx, %eax - xchgl %eax, (%rdi) /* NB: lock is implied */ - - testl %eax, %eax - jnz 1b - - popq %rdx - cfi_adjust_cfa_offset(-8) - cfi_restore(%rdx) - popq %r10 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r10) - retq - cfi_endproc - .size __lll_lock_wait_private,.-__lll_lock_wait_private - -#if !IS_IN (libc) - .globl __lll_lock_wait - .type __lll_lock_wait,@function - .hidden __lll_lock_wait - .align 16 -__lll_lock_wait: - cfi_startproc - pushq %r10 - cfi_adjust_cfa_offset(8) - pushq %rdx - cfi_adjust_cfa_offset(8) - cfi_offset(%r10, -16) - cfi_offset(%rdx, -24) - xorq %r10, %r10 /* No timeout. */ - movl $2, %edx - LOAD_FUTEX_WAIT (%esi) - - cmpl %edx, %eax /* NB: %edx == 2 */ - jne 2f - -1: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi) - movl $SYS_futex, %eax - syscall - -2: movl %edx, %eax - xchgl %eax, (%rdi) /* NB: lock is implied */ - - testl %eax, %eax - jnz 1b - - popq %rdx - cfi_adjust_cfa_offset(-8) - cfi_restore(%rdx) - popq %r10 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r10) - retq - cfi_endproc - .size __lll_lock_wait,.-__lll_lock_wait - - /* %rdi: futex - %rsi: flags - %rdx: timeout - %eax: futex value - */ - .globl __lll_timedlock_wait - .type __lll_timedlock_wait,@function - .hidden __lll_timedlock_wait - .align 16 -__lll_timedlock_wait: - cfi_startproc -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME -# ifdef PIC - cmpl $0, __have_futex_clock_realtime(%rip) -# else - cmpl $0, __have_futex_clock_realtime -# endif - je .Lreltmo -# endif - - cmpq $0, (%rdx) - js 5f - - pushq %r9 - cfi_adjust_cfa_offset(8) - cfi_rel_offset(%r9, 0) - - movq %rdx, %r10 - movl $0xffffffff, %r9d - LOAD_FUTEX_WAIT_ABS (%esi) - - movl $2, %edx - cmpl %edx, %eax - jne 2f - -1: movl $SYS_futex, %eax - movl $2, %edx - syscall - -2: xchgl %edx, (%rdi) /* NB: lock is implied */ - - testl %edx, %edx - jz 3f - - cmpl $-ETIMEDOUT, %eax - je 4f - cmpl $-EINVAL, %eax - jne 1b -4: movl %eax, %edx - negl %edx - -3: movl %edx, %eax - popq %r9 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r9) - retq - -5: movl $ETIMEDOUT, %eax - retq - -# ifndef __ASSUME_FUTEX_CLOCK_REALTIME -.Lreltmo: - /* Check for a valid timeout value. */ - cmpq $1000000000, 8(%rdx) - jae 3f - - pushq %r8 - cfi_adjust_cfa_offset(8) - pushq %r9 - cfi_adjust_cfa_offset(8) - pushq %r12 - cfi_adjust_cfa_offset(8) - pushq %r13 - cfi_adjust_cfa_offset(8) - pushq %r14 - cfi_adjust_cfa_offset(8) - cfi_offset(%r8, -16) - cfi_offset(%r9, -24) - cfi_offset(%r12, -32) - cfi_offset(%r13, -40) - cfi_offset(%r14, -48) - pushq %rsi - cfi_adjust_cfa_offset(8) - - /* Stack frame for the timespec and timeval structs. */ - subq $24, %rsp - cfi_adjust_cfa_offset(24) - - movq %rdi, %r12 - movq %rdx, %r13 - - movl $2, %edx - xchgl %edx, (%r12) - - testl %edx, %edx - je 6f - -1: - /* Get current time. */ - movq %rsp, %rdi - xorl %esi, %esi - /* This call works because we directly jump to a system call entry - which preserves all the registers. */ - call JUMPTARGET(__gettimeofday) - - /* Compute relative timeout. */ - movq 8(%rsp), %rax - movl $1000, %edi - mul %rdi /* Milli seconds to nano seconds. */ - movq (%r13), %rdi - movq 8(%r13), %rsi - subq (%rsp), %rdi - subq %rax, %rsi - jns 4f - addq $1000000000, %rsi - decq %rdi -4: testq %rdi, %rdi - js 2f /* Time is already up. */ - - /* Store relative timeout. */ - movq %rdi, (%rsp) - movq %rsi, 8(%rsp) - - /* Futex call. */ - movl $2, %edx - movl $1, %eax - movq %rsp, %r10 - movl 24(%rsp), %esi - LOAD_FUTEX_WAIT (%esi) - movq %r12, %rdi - movl $SYS_futex, %eax - syscall - - /* NB: %edx == 2 */ - xchgl %edx, (%r12) - - testl %edx, %edx - je 6f - - cmpl $-ETIMEDOUT, %eax - jne 1b -2: movl $ETIMEDOUT, %edx - -6: addq $32, %rsp - cfi_adjust_cfa_offset(-32) - popq %r14 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r14) - popq %r13 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r13) - popq %r12 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r12) - popq %r9 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r9) - popq %r8 - cfi_adjust_cfa_offset(-8) - cfi_restore(%r8) - movl %edx, %eax - retq - -3: movl $EINVAL, %eax - retq -# endif - cfi_endproc - .size __lll_timedlock_wait,.-__lll_timedlock_wait -#endif - - - .globl __lll_unlock_wake_private - .type __lll_unlock_wake_private,@function - .hidden __lll_unlock_wake_private - .align 16 -__lll_unlock_wake_private: - cfi_startproc - pushq %rsi - cfi_adjust_cfa_offset(8) - pushq %rdx - cfi_adjust_cfa_offset(8) - cfi_offset(%rsi, -16) - cfi_offset(%rdx, -24) - - movl $0, (%rdi) - LOAD_PRIVATE_FUTEX_WAKE (%esi) - movl $1, %edx /* Wake one thread. */ - movl $SYS_futex, %eax - syscall - - popq %rdx - cfi_adjust_cfa_offset(-8) - cfi_restore(%rdx) - popq %rsi - cfi_adjust_cfa_offset(-8) - cfi_restore(%rsi) - retq - cfi_endproc - .size __lll_unlock_wake_private,.-__lll_unlock_wake_private - -#if !IS_IN (libc) - .globl __lll_unlock_wake - .type __lll_unlock_wake,@function - .hidden __lll_unlock_wake - .align 16 -__lll_unlock_wake: - cfi_startproc - pushq %rsi - cfi_adjust_cfa_offset(8) - pushq %rdx - cfi_adjust_cfa_offset(8) - cfi_offset(%rsi, -16) - cfi_offset(%rdx, -24) - - movl $0, (%rdi) - LOAD_FUTEX_WAKE (%esi) - movl $1, %edx /* Wake one thread. */ - movl $SYS_futex, %eax - syscall - - popq %rdx - cfi_adjust_cfa_offset(-8) - cfi_restore(%rdx) - popq %rsi - cfi_adjust_cfa_offset(-8) - cfi_restore(%rsi) - retq - cfi_endproc - .size __lll_unlock_wake,.-__lll_unlock_wake -#endif diff --git a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h deleted file mode 100644 index 8cbc1caa5b..0000000000 --- a/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ /dev/null @@ -1,243 +0,0 @@ -/* Copyright (C) 2002-2019 Free Software Foundation, Inc. - This file is part of the GNU C Library. - Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#ifndef _LOWLEVELLOCK_H -#define _LOWLEVELLOCK_H 1 - -#ifndef __ASSEMBLER__ -# include <time.h> -# include <sys/param.h> -# include <bits/pthreadtypes.h> -# include <kernel-features.h> - -# ifndef LOCK_INSTR -# ifdef UP -# define LOCK_INSTR /* nothing */ -# else -# define LOCK_INSTR "lock;" -# endif -# endif -#else -# ifndef LOCK -# ifdef UP -# define LOCK -# else -# define LOCK lock -# endif -# endif -#endif - -#include <lowlevellock-futex.h> - -/* XXX Remove when no assembler code uses futexes anymore. */ -#define SYS_futex __NR_futex - -#ifndef __ASSEMBLER__ - -/* Initializer for lock. */ -#define LLL_LOCK_INITIALIZER (0) -#define LLL_LOCK_INITIALIZER_LOCKED (1) -#define LLL_LOCK_INITIALIZER_WAITERS (2) - - -/* NB: in the lll_trylock macro we simply return the value in %eax - after the cmpxchg instruction. In case the operation succeded this - value is zero. In case the operation failed, the cmpxchg instruction - has loaded the current value of the memory work which is guaranteed - to be nonzero. */ -#if !IS_IN (libc) || defined UP -# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" -#else -# define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ - "je 0f\n\t" \ - "lock; cmpxchgl %2, %1\n\t" \ - "jmp 1f\n\t" \ - "0:\tcmpxchgl %2, %1\n\t" \ - "1:" -#endif - -#define lll_trylock(futex) \ - ({ int ret; \ - __asm __volatile (__lll_trylock_asm \ - : "=a" (ret), "=m" (futex) \ - : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ - "0" (LLL_LOCK_INITIALIZER) \ - : "memory"); \ - ret; }) - -#define lll_cond_trylock(futex) \ - ({ int ret; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ - : "=a" (ret), "=m" (futex) \ - : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ - "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ - : "memory"); \ - ret; }) - -#if !IS_IN (libc) || defined UP -# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \ - "jz 24f\n\t" -#else -# define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ - "je 0f\n\t" \ - "lock; cmpxchgl %4, %2\n\t" \ - "jnz 1f\n\t" \ - "jmp 24f\n" \ - "0:\tcmpxchgl %4, %2\n\t" \ - "jz 24f\n\t" -#endif - -#define lll_lock(futex, private) \ - (void) \ - ({ int ignore1, ignore2, ignore3; \ - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ - __asm __volatile (__lll_lock_asm_start \ - "1:\tlea %2, %%" RDI_LP "\n" \ - "2:\tsub $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset 128\n" \ - "3:\tcallq __lll_lock_wait_private\n" \ - "4:\tadd $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset -128\n" \ - "24:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ - "=a" (ignore3) \ - : "0" (1), "m" (futex), "3" (0) \ - : "cx", "r11", "cc", "memory"); \ - else \ - __asm __volatile (__lll_lock_asm_start \ - "1:\tlea %2, %%" RDI_LP "\n" \ - "2:\tsub $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset 128\n" \ - "3:\tcallq __lll_lock_wait\n" \ - "4:\tadd $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset -128\n" \ - "24:" \ - : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ - "=a" (ignore3) \ - : "1" (1), "m" (futex), "3" (0), "0" (private) \ - : "cx", "r11", "cc", "memory"); \ - }) \ - -#define lll_cond_lock(futex, private) \ - (void) \ - ({ int ignore1, ignore2, ignore3; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ - "jz 24f\n" \ - "1:\tlea %2, %%" RDI_LP "\n" \ - "2:\tsub $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset 128\n" \ - "3:\tcallq __lll_lock_wait\n" \ - "4:\tadd $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset -128\n" \ - "24:" \ - : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ - "=a" (ignore3) \ - : "1" (2), "m" (futex), "3" (0), "0" (private) \ - : "cx", "r11", "cc", "memory"); \ - }) - -#define lll_timedlock(futex, timeout, private) \ - ({ int result, ignore1, ignore2, ignore3; \ - __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ - "jz 24f\n" \ - "1:\tlea %4, %%" RDI_LP "\n" \ - "0:\tmov %8, %%" RDX_LP "\n" \ - "2:\tsub $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset 128\n" \ - "3:\tcallq __lll_timedlock_wait\n" \ - "4:\tadd $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset -128\n" \ - "24:" \ - : "=a" (result), "=D" (ignore1), "=S" (ignore2), \ - "=&d" (ignore3), "=m" (futex) \ - : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ - "2" (private) \ - : "memory", "cx", "cc", "r10", "r11"); \ - result; }) - -extern int __lll_timedlock_elision (int *futex, short *adapt_count, - const struct timespec *timeout, - int private) attribute_hidden; - -#define lll_timedlock_elision(futex, adapt_count, timeout, private) \ - __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) - -#if !IS_IN (libc) || defined UP -# define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \ - "je 24f\n\t" -#else -# define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ - "je 0f\n\t" \ - "lock; decl %0\n\t" \ - "jne 1f\n\t" \ - "jmp 24f\n\t" \ - "0:\tdecl %0\n\t" \ - "je 24f\n\t" -#endif - -#define lll_unlock(futex, private) \ - (void) \ - ({ int ignore; \ - if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ - __asm __volatile (__lll_unlock_asm_start \ - "1:\tlea %0, %%" RDI_LP "\n" \ - "2:\tsub $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset 128\n" \ - "3:\tcallq __lll_unlock_wake_private\n" \ - "4:\tadd $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset -128\n" \ - "24:" \ - : "=m" (futex), "=&D" (ignore) \ - : "m" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); \ - else \ - __asm __volatile (__lll_unlock_asm_start \ - "1:\tlea %0, %%" RDI_LP "\n" \ - "2:\tsub $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset 128\n" \ - "3:\tcallq __lll_unlock_wake\n" \ - "4:\tadd $128, %%" RSP_LP "\n" \ - ".cfi_adjust_cfa_offset -128\n" \ - "24:" \ - : "=m" (futex), "=&D" (ignore) \ - : "m" (futex), "S" (private) \ - : "ax", "cx", "r11", "cc", "memory"); \ - }) - -#define lll_islocked(futex) \ - (futex != LLL_LOCK_INITIALIZER) - -extern int __lll_lock_elision (int *futex, short *adapt_count, int private) - attribute_hidden; - -extern int __lll_unlock_elision (int *lock, int private) - attribute_hidden; - -extern int __lll_trylock_elision (int *lock, short *adapt_count) - attribute_hidden; - -#define lll_lock_elision(futex, adapt_count, private) \ - __lll_lock_elision (&(futex), &(adapt_count), private) -#define lll_unlock_elision(futex, adapt_count, private) \ - __lll_unlock_elision (&(futex), private) -#define lll_trylock_elision(futex, adapt_count) \ - __lll_trylock_elision (&(futex), &(adapt_count)) - -#endif /* !__ASSEMBLER__ */ - -#endif /* lowlevellock.h */