You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
444 lines
11 KiB
444 lines
11 KiB
4 years ago
|
This patch is based on the below upstream commit.
|
||
|
File deletions were altered to reflect file renames.
|
||
|
|
||
|
commit f50277c19df0937ea9691ab7e7c642ecd3ed3d94
|
||
|
Author: Torvald Riegel <triegel@redhat.com>
|
||
|
Date: Sun Oct 19 21:59:26 2014 +0200
|
||
|
|
||
|
pthread_once: Add fast path and remove x86 variants.
|
||
|
|
||
|
diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c
|
||
|
index 595bd7e298003e00..2afb79c01fe0a61e 100644
|
||
|
--- a/nptl/pthread_once.c
|
||
|
+++ b/nptl/pthread_once.c
|
||
|
@@ -58,11 +58,13 @@ clear_once_control (void *arg)
|
||
|
initialization is interrupted, we then fork 2^30 times (30 bits of
|
||
|
once_control are used for the fork generation), and try to initialize
|
||
|
again, we can deadlock because we can't distinguish the in-progress and
|
||
|
- interrupted cases anymore. */
|
||
|
-int
|
||
|
-__pthread_once (once_control, init_routine)
|
||
|
- pthread_once_t *once_control;
|
||
|
- void (*init_routine) (void);
|
||
|
+ interrupted cases anymore.
|
||
|
+ XXX: We split out this slow path because current compilers do not generate
|
||
|
+ as efficient code when the fast path in __pthread_once below is not in a
|
||
|
+ separate function. */
|
||
|
+static int
|
||
|
+__attribute__ ((noinline))
|
||
|
+__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
|
||
|
{
|
||
|
while (1)
|
||
|
{
|
||
|
@@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine)
|
||
|
signals that initialization has finished, we need to see any
|
||
|
data modifications done during initialization. */
|
||
|
val = *once_control;
|
||
|
- atomic_read_barrier();
|
||
|
+ atomic_read_barrier ();
|
||
|
do
|
||
|
{
|
||
|
/* Check if the initialization has already been done. */
|
||
|
@@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine)
|
||
|
|
||
|
return 0;
|
||
|
}
|
||
|
+
|
||
|
+int
|
||
|
+__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
|
||
|
+{
|
||
|
+ /* Fast path. See __pthread_once_slow. */
|
||
|
+ int val;
|
||
|
+ val = *once_control;
|
||
|
+ atomic_read_barrier ();
|
||
|
+ if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
|
||
|
+ return 0;
|
||
|
+ else
|
||
|
+ return __pthread_once_slow (once_control, init_routine);
|
||
|
+}
|
||
|
weak_alias (__pthread_once, pthread_once)
|
||
|
hidden_def (__pthread_once)
|
||
|
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
|
||
|
deleted file mode 100644
|
||
|
index ca3b860a7f6f95ae..0000000000000000
|
||
|
--- a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
|
||
|
+++ /dev/null
|
||
|
@@ -1,178 +0,0 @@
|
||
|
-/* Copyright (C) 2002-2012 Free Software Foundation, Inc.
|
||
|
- This file is part of the GNU C Library.
|
||
|
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
|
||
|
-
|
||
|
- The GNU C Library is free software; you can redistribute it and/or
|
||
|
- modify it under the terms of the GNU Lesser General Public
|
||
|
- License as published by the Free Software Foundation; either
|
||
|
- version 2.1 of the License, or (at your option) any later version.
|
||
|
-
|
||
|
- The GNU C Library is distributed in the hope that it will be useful,
|
||
|
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||
|
- Lesser General Public License for more details.
|
||
|
-
|
||
|
- You should have received a copy of the GNU Lesser General Public
|
||
|
- License along with the GNU C Library; if not, see
|
||
|
- <http://www.gnu.org/licenses/>. */
|
||
|
-
|
||
|
-#include <unwindbuf.h>
|
||
|
-#include <sysdep.h>
|
||
|
-#include <kernel-features.h>
|
||
|
-#include <lowlevellock.h>
|
||
|
-
|
||
|
-
|
||
|
- .comm __fork_generation, 4, 4
|
||
|
-
|
||
|
- .text
|
||
|
-
|
||
|
-
|
||
|
- .globl __pthread_once
|
||
|
- .type __pthread_once,@function
|
||
|
- .align 16
|
||
|
- cfi_startproc
|
||
|
-__pthread_once:
|
||
|
- movl 4(%esp), %ecx
|
||
|
- testl $2, (%ecx)
|
||
|
- jz 1f
|
||
|
- xorl %eax, %eax
|
||
|
- ret
|
||
|
-
|
||
|
-1: pushl %ebx
|
||
|
- cfi_adjust_cfa_offset (4)
|
||
|
- cfi_rel_offset (3, 0)
|
||
|
- pushl %esi
|
||
|
- cfi_adjust_cfa_offset (4)
|
||
|
- cfi_rel_offset (6, 0)
|
||
|
- movl %ecx, %ebx
|
||
|
- xorl %esi, %esi
|
||
|
-
|
||
|
- /* Not yet initialized or initialization in progress.
|
||
|
- Get the fork generation counter now. */
|
||
|
-6: movl (%ebx), %eax
|
||
|
-#ifdef PIC
|
||
|
- LOAD_PIC_REG(cx)
|
||
|
-#endif
|
||
|
-
|
||
|
-5: movl %eax, %edx
|
||
|
-
|
||
|
- testl $2, %eax
|
||
|
- jnz 4f
|
||
|
-
|
||
|
- andl $3, %edx
|
||
|
-#ifdef PIC
|
||
|
- orl __fork_generation@GOTOFF(%ecx), %edx
|
||
|
-#else
|
||
|
- orl __fork_generation, %edx
|
||
|
-#endif
|
||
|
- orl $1, %edx
|
||
|
-
|
||
|
- LOCK
|
||
|
- cmpxchgl %edx, (%ebx)
|
||
|
- jnz 5b
|
||
|
-
|
||
|
- /* Check whether another thread already runs the initializer. */
|
||
|
- testl $1, %eax
|
||
|
- jz 3f /* No -> do it. */
|
||
|
-
|
||
|
- /* Check whether the initializer execution was interrupted
|
||
|
- by a fork. */
|
||
|
- xorl %edx, %eax
|
||
|
- testl $0xfffffffc, %eax
|
||
|
- jnz 3f /* Different for generation -> run initializer. */
|
||
|
-
|
||
|
- /* Somebody else got here first. Wait. */
|
||
|
-#ifdef __ASSUME_PRIVATE_FUTEX
|
||
|
- movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
|
||
|
-#else
|
||
|
-# if FUTEX_WAIT == 0
|
||
|
- movl %gs:PRIVATE_FUTEX, %ecx
|
||
|
-# else
|
||
|
- movl $FUTEX_WAIT, %ecx
|
||
|
- orl %gs:PRIVATE_FUTEX, %ecx
|
||
|
-# endif
|
||
|
-#endif
|
||
|
- movl $SYS_futex, %eax
|
||
|
- ENTER_KERNEL
|
||
|
- jmp 6b
|
||
|
-
|
||
|
-3: /* Call the initializer function after setting up the
|
||
|
- cancellation handler. Note that it is not possible here
|
||
|
- to use the unwind-based cleanup handling. This would require
|
||
|
- that the user-provided function and all the code it calls
|
||
|
- is compiled with exceptions. Unfortunately this cannot be
|
||
|
- guaranteed. */
|
||
|
- subl $UNWINDBUFSIZE+8, %esp
|
||
|
- cfi_adjust_cfa_offset (UNWINDBUFSIZE+8)
|
||
|
- movl %ecx, %ebx /* PIC register value. */
|
||
|
-
|
||
|
- leal 8+UWJMPBUF(%esp), %eax
|
||
|
- movl $0, 4(%esp)
|
||
|
- movl %eax, (%esp)
|
||
|
- call __sigsetjmp@PLT
|
||
|
- testl %eax, %eax
|
||
|
- jne 7f
|
||
|
-
|
||
|
- leal 8(%esp), %eax
|
||
|
- call HIDDEN_JUMPTARGET(__pthread_register_cancel)
|
||
|
-
|
||
|
- /* Call the user-provided initialization function. */
|
||
|
- call *24+UNWINDBUFSIZE(%esp)
|
||
|
-
|
||
|
- /* Pop the cleanup handler. */
|
||
|
- leal 8(%esp), %eax
|
||
|
- call HIDDEN_JUMPTARGET(__pthread_unregister_cancel)
|
||
|
- addl $UNWINDBUFSIZE+8, %esp
|
||
|
- cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8)
|
||
|
-
|
||
|
- /* Sucessful run of the initializer. Signal that we are done. */
|
||
|
- movl 12(%esp), %ebx
|
||
|
- LOCK
|
||
|
- addl $1, (%ebx)
|
||
|
-
|
||
|
- /* Wake up all other threads. */
|
||
|
- movl $0x7fffffff, %edx
|
||
|
-#ifdef __ASSUME_PRIVATE_FUTEX
|
||
|
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
|
||
|
-#else
|
||
|
- movl $FUTEX_WAKE, %ecx
|
||
|
- orl %gs:PRIVATE_FUTEX, %ecx
|
||
|
-#endif
|
||
|
- movl $SYS_futex, %eax
|
||
|
- ENTER_KERNEL
|
||
|
-
|
||
|
-4: popl %esi
|
||
|
- cfi_adjust_cfa_offset (-4)
|
||
|
- cfi_restore (6)
|
||
|
- popl %ebx
|
||
|
- cfi_adjust_cfa_offset (-4)
|
||
|
- cfi_restore (3)
|
||
|
- xorl %eax, %eax
|
||
|
- ret
|
||
|
-
|
||
|
-7: /* __sigsetjmp returned for the second time. */
|
||
|
- movl 20+UNWINDBUFSIZE(%esp), %ebx
|
||
|
- cfi_adjust_cfa_offset (UNWINDBUFSIZE+16)
|
||
|
- cfi_offset (3, -8)
|
||
|
- cfi_offset (6, -12)
|
||
|
- movl $0, (%ebx)
|
||
|
-
|
||
|
- movl $0x7fffffff, %edx
|
||
|
-#ifdef __ASSUME_PRIVATE_FUTEX
|
||
|
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
|
||
|
-#else
|
||
|
- movl $FUTEX_WAKE, %ecx
|
||
|
- orl %gs:PRIVATE_FUTEX, %ecx
|
||
|
-#endif
|
||
|
- movl $SYS_futex, %eax
|
||
|
- ENTER_KERNEL
|
||
|
-
|
||
|
- leal 8(%esp), %eax
|
||
|
- call HIDDEN_JUMPTARGET (__pthread_unwind_next)
|
||
|
- /* NOTREACHED */
|
||
|
- hlt
|
||
|
- cfi_endproc
|
||
|
- .size __pthread_once,.-__pthread_once
|
||
|
-
|
||
|
-hidden_def (__pthread_once)
|
||
|
-strong_alias (__pthread_once, pthread_once)
|
||
|
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
|
||
|
deleted file mode 100644
|
||
|
index 7f5c0810fa16b987..0000000000000000
|
||
|
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
|
||
|
+++ /dev/null
|
||
|
@@ -1,193 +0,0 @@
|
||
|
-/* Copyright (C) 2002-2012 Free Software Foundation, Inc.
|
||
|
- This file is part of the GNU C Library.
|
||
|
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
|
||
|
-
|
||
|
- The GNU C Library is free software; you can redistribute it and/or
|
||
|
- modify it under the terms of the GNU Lesser General Public
|
||
|
- License as published by the Free Software Foundation; either
|
||
|
- version 2.1 of the License, or (at your option) any later version.
|
||
|
-
|
||
|
- The GNU C Library is distributed in the hope that it will be useful,
|
||
|
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||
|
- Lesser General Public License for more details.
|
||
|
-
|
||
|
- You should have received a copy of the GNU Lesser General Public
|
||
|
- License along with the GNU C Library; if not, see
|
||
|
- <http://www.gnu.org/licenses/>. */
|
||
|
-
|
||
|
-#include <sysdep.h>
|
||
|
-#include <kernel-features.h>
|
||
|
-#include <tcb-offsets.h>
|
||
|
-#include <lowlevellock.h>
|
||
|
-
|
||
|
-
|
||
|
- .comm __fork_generation, 4, 4
|
||
|
-
|
||
|
- .text
|
||
|
-
|
||
|
-
|
||
|
- .globl __pthread_once
|
||
|
- .type __pthread_once,@function
|
||
|
- .align 16
|
||
|
-__pthread_once:
|
||
|
-.LSTARTCODE:
|
||
|
- cfi_startproc
|
||
|
-#ifdef SHARED
|
||
|
- cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
|
||
|
- DW.ref.__gcc_personality_v0)
|
||
|
- cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
|
||
|
-#else
|
||
|
- cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
|
||
|
- cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
|
||
|
-#endif
|
||
|
- testl $2, (%rdi)
|
||
|
- jz 1f
|
||
|
- xorl %eax, %eax
|
||
|
- retq
|
||
|
-
|
||
|
- /* Preserve the function pointer. */
|
||
|
-1: pushq %rsi
|
||
|
- cfi_adjust_cfa_offset(8)
|
||
|
- xorq %r10, %r10
|
||
|
-
|
||
|
- /* Not yet initialized or initialization in progress.
|
||
|
- Get the fork generation counter now. */
|
||
|
-6: movl (%rdi), %eax
|
||
|
-
|
||
|
-5: movl %eax, %edx
|
||
|
-
|
||
|
- testl $2, %eax
|
||
|
- jnz 4f
|
||
|
-
|
||
|
- andl $3, %edx
|
||
|
- orl __fork_generation(%rip), %edx
|
||
|
- orl $1, %edx
|
||
|
-
|
||
|
- LOCK
|
||
|
- cmpxchgl %edx, (%rdi)
|
||
|
- jnz 5b
|
||
|
-
|
||
|
- /* Check whether another thread already runs the initializer. */
|
||
|
- testl $1, %eax
|
||
|
- jz 3f /* No -> do it. */
|
||
|
-
|
||
|
- /* Check whether the initializer execution was interrupted
|
||
|
- by a fork. */
|
||
|
- xorl %edx, %eax
|
||
|
- testl $0xfffffffc, %eax
|
||
|
- jnz 3f /* Different for generation -> run initializer. */
|
||
|
-
|
||
|
- /* Somebody else got here first. Wait. */
|
||
|
-#ifdef __ASSUME_PRIVATE_FUTEX
|
||
|
- movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
|
||
|
-#else
|
||
|
-# if FUTEX_WAIT == 0
|
||
|
- movl %fs:PRIVATE_FUTEX, %esi
|
||
|
-# else
|
||
|
- movl $FUTEX_WAIT, %esi
|
||
|
- orl %fs:PRIVATE_FUTEX, %esi
|
||
|
-# endif
|
||
|
-#endif
|
||
|
- movl $SYS_futex, %eax
|
||
|
- syscall
|
||
|
- jmp 6b
|
||
|
-
|
||
|
- /* Preserve the pointer to the control variable. */
|
||
|
-3: pushq %rdi
|
||
|
- cfi_adjust_cfa_offset(8)
|
||
|
- pushq %rdi
|
||
|
- cfi_adjust_cfa_offset(8)
|
||
|
-
|
||
|
-.LcleanupSTART:
|
||
|
- callq *16(%rsp)
|
||
|
-.LcleanupEND:
|
||
|
-
|
||
|
- /* Get the control variable address back. */
|
||
|
- popq %rdi
|
||
|
- cfi_adjust_cfa_offset(-8)
|
||
|
-
|
||
|
- /* Sucessful run of the initializer. Signal that we are done. */
|
||
|
- LOCK
|
||
|
- incl (%rdi)
|
||
|
-
|
||
|
- addq $8, %rsp
|
||
|
- cfi_adjust_cfa_offset(-8)
|
||
|
-
|
||
|
- /* Wake up all other threads. */
|
||
|
- movl $0x7fffffff, %edx
|
||
|
-#ifdef __ASSUME_PRIVATE_FUTEX
|
||
|
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
|
||
|
-#else
|
||
|
- movl $FUTEX_WAKE, %esi
|
||
|
- orl %fs:PRIVATE_FUTEX, %esi
|
||
|
-#endif
|
||
|
- movl $SYS_futex, %eax
|
||
|
- syscall
|
||
|
-
|
||
|
-4: addq $8, %rsp
|
||
|
- cfi_adjust_cfa_offset(-8)
|
||
|
- xorl %eax, %eax
|
||
|
- retq
|
||
|
- .size __pthread_once,.-__pthread_once
|
||
|
-
|
||
|
-
|
||
|
-hidden_def (__pthread_once)
|
||
|
-strong_alias (__pthread_once, pthread_once)
|
||
|
-
|
||
|
-
|
||
|
- .type clear_once_control,@function
|
||
|
- .align 16
|
||
|
-clear_once_control:
|
||
|
- cfi_adjust_cfa_offset(3 * 8)
|
||
|
- movq (%rsp), %rdi
|
||
|
- movq %rax, %r8
|
||
|
- movl $0, (%rdi)
|
||
|
-
|
||
|
- movl $0x7fffffff, %edx
|
||
|
-#ifdef __ASSUME_PRIVATE_FUTEX
|
||
|
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
|
||
|
-#else
|
||
|
- movl $FUTEX_WAKE, %esi
|
||
|
- orl %fs:PRIVATE_FUTEX, %esi
|
||
|
-#endif
|
||
|
- movl $SYS_futex, %eax
|
||
|
- syscall
|
||
|
-
|
||
|
- movq %r8, %rdi
|
||
|
-.LcallUR:
|
||
|
- call _Unwind_Resume@PLT
|
||
|
- hlt
|
||
|
-.LENDCODE:
|
||
|
- cfi_endproc
|
||
|
- .size clear_once_control,.-clear_once_control
|
||
|
-
|
||
|
-
|
||
|
- .section .gcc_except_table,"a",@progbits
|
||
|
-.LexceptSTART:
|
||
|
- .byte DW_EH_PE_omit # @LPStart format
|
||
|
- .byte DW_EH_PE_omit # @TType format
|
||
|
- .byte DW_EH_PE_uleb128 # call-site format
|
||
|
- .uleb128 .Lcstend-.Lcstbegin
|
||
|
-.Lcstbegin:
|
||
|
- .uleb128 .LcleanupSTART-.LSTARTCODE
|
||
|
- .uleb128 .LcleanupEND-.LcleanupSTART
|
||
|
- .uleb128 clear_once_control-.LSTARTCODE
|
||
|
- .uleb128 0
|
||
|
- .uleb128 .LcallUR-.LSTARTCODE
|
||
|
- .uleb128 .LENDCODE-.LcallUR
|
||
|
- .uleb128 0
|
||
|
- .uleb128 0
|
||
|
-.Lcstend:
|
||
|
-
|
||
|
-
|
||
|
-#ifdef SHARED
|
||
|
- .hidden DW.ref.__gcc_personality_v0
|
||
|
- .weak DW.ref.__gcc_personality_v0
|
||
|
- .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
|
||
|
- .align LP_SIZE
|
||
|
- .type DW.ref.__gcc_personality_v0, @object
|
||
|
- .size DW.ref.__gcc_personality_v0, LP_SIZE
|
||
|
-DW.ref.__gcc_personality_v0:
|
||
|
- ASM_ADDR __gcc_personality_v0
|
||
|
-#endif
|