You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

444 lines
11 KiB

This patch is based on the below upstream commit.
File deletions were altered to reflect file renames.
commit f50277c19df0937ea9691ab7e7c642ecd3ed3d94
Author: Torvald Riegel <triegel@redhat.com>
Date: Sun Oct 19 21:59:26 2014 +0200
pthread_once: Add fast path and remove x86 variants.
diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c
index 595bd7e298003e00..2afb79c01fe0a61e 100644
--- a/nptl/pthread_once.c
+++ b/nptl/pthread_once.c
@@ -58,11 +58,13 @@ clear_once_control (void *arg)
initialization is interrupted, we then fork 2^30 times (30 bits of
once_control are used for the fork generation), and try to initialize
again, we can deadlock because we can't distinguish the in-progress and
- interrupted cases anymore. */
-int
-__pthread_once (once_control, init_routine)
- pthread_once_t *once_control;
- void (*init_routine) (void);
+ interrupted cases anymore.
+ XXX: We split out this slow path because current compilers do not generate
+ as efficient code when the fast path in __pthread_once below is not in a
+ separate function. */
+static int
+__attribute__ ((noinline))
+__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
{
while (1)
{
@@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine)
signals that initialization has finished, we need to see any
data modifications done during initialization. */
val = *once_control;
- atomic_read_barrier();
+ atomic_read_barrier ();
do
{
/* Check if the initialization has already been done. */
@@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine)
return 0;
}
+
+int
+__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
+{
+ /* Fast path. See __pthread_once_slow. */
+ int val;
+ val = *once_control;
+ atomic_read_barrier ();
+ if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
+ return 0;
+ else
+ return __pthread_once_slow (once_control, init_routine);
+}
weak_alias (__pthread_once, pthread_once)
hidden_def (__pthread_once)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
deleted file mode 100644
index ca3b860a7f6f95ae..0000000000000000
--- a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
+++ /dev/null
@@ -1,178 +0,0 @@
-/* Copyright (C) 2002-2012 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <unwindbuf.h>
-#include <sysdep.h>
-#include <kernel-features.h>
-#include <lowlevellock.h>
-
-
- .comm __fork_generation, 4, 4
-
- .text
-
-
- .globl __pthread_once
- .type __pthread_once,@function
- .align 16
- cfi_startproc
-__pthread_once:
- movl 4(%esp), %ecx
- testl $2, (%ecx)
- jz 1f
- xorl %eax, %eax
- ret
-
-1: pushl %ebx
- cfi_adjust_cfa_offset (4)
- cfi_rel_offset (3, 0)
- pushl %esi
- cfi_adjust_cfa_offset (4)
- cfi_rel_offset (6, 0)
- movl %ecx, %ebx
- xorl %esi, %esi
-
- /* Not yet initialized or initialization in progress.
- Get the fork generation counter now. */
-6: movl (%ebx), %eax
-#ifdef PIC
- LOAD_PIC_REG(cx)
-#endif
-
-5: movl %eax, %edx
-
- testl $2, %eax
- jnz 4f
-
- andl $3, %edx
-#ifdef PIC
- orl __fork_generation@GOTOFF(%ecx), %edx
-#else
- orl __fork_generation, %edx
-#endif
- orl $1, %edx
-
- LOCK
- cmpxchgl %edx, (%ebx)
- jnz 5b
-
- /* Check whether another thread already runs the initializer. */
- testl $1, %eax
- jz 3f /* No -> do it. */
-
- /* Check whether the initializer execution was interrupted
- by a fork. */
- xorl %edx, %eax
- testl $0xfffffffc, %eax
- jnz 3f /* Different for generation -> run initializer. */
-
- /* Somebody else got here first. Wait. */
-#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
-#else
-# if FUTEX_WAIT == 0
- movl %gs:PRIVATE_FUTEX, %ecx
-# else
- movl $FUTEX_WAIT, %ecx
- orl %gs:PRIVATE_FUTEX, %ecx
-# endif
-#endif
- movl $SYS_futex, %eax
- ENTER_KERNEL
- jmp 6b
-
-3: /* Call the initializer function after setting up the
- cancellation handler. Note that it is not possible here
- to use the unwind-based cleanup handling. This would require
- that the user-provided function and all the code it calls
- is compiled with exceptions. Unfortunately this cannot be
- guaranteed. */
- subl $UNWINDBUFSIZE+8, %esp
- cfi_adjust_cfa_offset (UNWINDBUFSIZE+8)
- movl %ecx, %ebx /* PIC register value. */
-
- leal 8+UWJMPBUF(%esp), %eax
- movl $0, 4(%esp)
- movl %eax, (%esp)
- call __sigsetjmp@PLT
- testl %eax, %eax
- jne 7f
-
- leal 8(%esp), %eax
- call HIDDEN_JUMPTARGET(__pthread_register_cancel)
-
- /* Call the user-provided initialization function. */
- call *24+UNWINDBUFSIZE(%esp)
-
- /* Pop the cleanup handler. */
- leal 8(%esp), %eax
- call HIDDEN_JUMPTARGET(__pthread_unregister_cancel)
- addl $UNWINDBUFSIZE+8, %esp
- cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8)
-
- /* Sucessful run of the initializer. Signal that we are done. */
- movl 12(%esp), %ebx
- LOCK
- addl $1, (%ebx)
-
- /* Wake up all other threads. */
- movl $0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
-#else
- movl $FUTEX_WAKE, %ecx
- orl %gs:PRIVATE_FUTEX, %ecx
-#endif
- movl $SYS_futex, %eax
- ENTER_KERNEL
-
-4: popl %esi
- cfi_adjust_cfa_offset (-4)
- cfi_restore (6)
- popl %ebx
- cfi_adjust_cfa_offset (-4)
- cfi_restore (3)
- xorl %eax, %eax
- ret
-
-7: /* __sigsetjmp returned for the second time. */
- movl 20+UNWINDBUFSIZE(%esp), %ebx
- cfi_adjust_cfa_offset (UNWINDBUFSIZE+16)
- cfi_offset (3, -8)
- cfi_offset (6, -12)
- movl $0, (%ebx)
-
- movl $0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
-#else
- movl $FUTEX_WAKE, %ecx
- orl %gs:PRIVATE_FUTEX, %ecx
-#endif
- movl $SYS_futex, %eax
- ENTER_KERNEL
-
- leal 8(%esp), %eax
- call HIDDEN_JUMPTARGET (__pthread_unwind_next)
- /* NOTREACHED */
- hlt
- cfi_endproc
- .size __pthread_once,.-__pthread_once
-
-hidden_def (__pthread_once)
-strong_alias (__pthread_once, pthread_once)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
deleted file mode 100644
index 7f5c0810fa16b987..0000000000000000
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
+++ /dev/null
@@ -1,193 +0,0 @@
-/* Copyright (C) 2002-2012 Free Software Foundation, Inc.
- This file is part of the GNU C Library.
- Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
- The GNU C Library is free software; you can redistribute it and/or
- modify it under the terms of the GNU Lesser General Public
- License as published by the Free Software Foundation; either
- version 2.1 of the License, or (at your option) any later version.
-
- The GNU C Library is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- Lesser General Public License for more details.
-
- You should have received a copy of the GNU Lesser General Public
- License along with the GNU C Library; if not, see
- <http://www.gnu.org/licenses/>. */
-
-#include <sysdep.h>
-#include <kernel-features.h>
-#include <tcb-offsets.h>
-#include <lowlevellock.h>
-
-
- .comm __fork_generation, 4, 4
-
- .text
-
-
- .globl __pthread_once
- .type __pthread_once,@function
- .align 16
-__pthread_once:
-.LSTARTCODE:
- cfi_startproc
-#ifdef SHARED
- cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
- DW.ref.__gcc_personality_v0)
- cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
-#else
- cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
- cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
-#endif
- testl $2, (%rdi)
- jz 1f
- xorl %eax, %eax
- retq
-
- /* Preserve the function pointer. */
-1: pushq %rsi
- cfi_adjust_cfa_offset(8)
- xorq %r10, %r10
-
- /* Not yet initialized or initialization in progress.
- Get the fork generation counter now. */
-6: movl (%rdi), %eax
-
-5: movl %eax, %edx
-
- testl $2, %eax
- jnz 4f
-
- andl $3, %edx
- orl __fork_generation(%rip), %edx
- orl $1, %edx
-
- LOCK
- cmpxchgl %edx, (%rdi)
- jnz 5b
-
- /* Check whether another thread already runs the initializer. */
- testl $1, %eax
- jz 3f /* No -> do it. */
-
- /* Check whether the initializer execution was interrupted
- by a fork. */
- xorl %edx, %eax
- testl $0xfffffffc, %eax
- jnz 3f /* Different for generation -> run initializer. */
-
- /* Somebody else got here first. Wait. */
-#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
-#else
-# if FUTEX_WAIT == 0
- movl %fs:PRIVATE_FUTEX, %esi
-# else
- movl $FUTEX_WAIT, %esi
- orl %fs:PRIVATE_FUTEX, %esi
-# endif
-#endif
- movl $SYS_futex, %eax
- syscall
- jmp 6b
-
- /* Preserve the pointer to the control variable. */
-3: pushq %rdi
- cfi_adjust_cfa_offset(8)
- pushq %rdi
- cfi_adjust_cfa_offset(8)
-
-.LcleanupSTART:
- callq *16(%rsp)
-.LcleanupEND:
-
- /* Get the control variable address back. */
- popq %rdi
- cfi_adjust_cfa_offset(-8)
-
- /* Sucessful run of the initializer. Signal that we are done. */
- LOCK
- incl (%rdi)
-
- addq $8, %rsp
- cfi_adjust_cfa_offset(-8)
-
- /* Wake up all other threads. */
- movl $0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
-#else
- movl $FUTEX_WAKE, %esi
- orl %fs:PRIVATE_FUTEX, %esi
-#endif
- movl $SYS_futex, %eax
- syscall
-
-4: addq $8, %rsp
- cfi_adjust_cfa_offset(-8)
- xorl %eax, %eax
- retq
- .size __pthread_once,.-__pthread_once
-
-
-hidden_def (__pthread_once)
-strong_alias (__pthread_once, pthread_once)
-
-
- .type clear_once_control,@function
- .align 16
-clear_once_control:
- cfi_adjust_cfa_offset(3 * 8)
- movq (%rsp), %rdi
- movq %rax, %r8
- movl $0, (%rdi)
-
- movl $0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
- movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
-#else
- movl $FUTEX_WAKE, %esi
- orl %fs:PRIVATE_FUTEX, %esi
-#endif
- movl $SYS_futex, %eax
- syscall
-
- movq %r8, %rdi
-.LcallUR:
- call _Unwind_Resume@PLT
- hlt
-.LENDCODE:
- cfi_endproc
- .size clear_once_control,.-clear_once_control
-
-
- .section .gcc_except_table,"a",@progbits
-.LexceptSTART:
- .byte DW_EH_PE_omit # @LPStart format
- .byte DW_EH_PE_omit # @TType format
- .byte DW_EH_PE_uleb128 # call-site format
- .uleb128 .Lcstend-.Lcstbegin
-.Lcstbegin:
- .uleb128 .LcleanupSTART-.LSTARTCODE
- .uleb128 .LcleanupEND-.LcleanupSTART
- .uleb128 clear_once_control-.LSTARTCODE
- .uleb128 0
- .uleb128 .LcallUR-.LSTARTCODE
- .uleb128 .LENDCODE-.LcallUR
- .uleb128 0
- .uleb128 0
-.Lcstend:
-
-
-#ifdef SHARED
- .hidden DW.ref.__gcc_personality_v0
- .weak DW.ref.__gcc_personality_v0
- .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
- .align LP_SIZE
- .type DW.ref.__gcc_personality_v0, @object
- .size DW.ref.__gcc_personality_v0, LP_SIZE
-DW.ref.__gcc_personality_v0:
- ASM_ADDR __gcc_personality_v0
-#endif