From cca51a74315c37614042113b004505b150d305d7 Mon Sep 17 00:00:00 2001 From: Stefan Liebler Date: Thu, 8 Oct 2015 10:54:21 +0200 Subject: [PATCH 10/30] S390: Optimize strlen and wcslen. upstream-commit-id: 9472f35a0a6dbbda82ce103aaf0f5013f5d46e34 https://www.sourceware.org/ml/libc-alpha/2015-07/msg00089.html This patch provides optimized versions of strlen and wcslen with the z13 vector instructions. The helper macro IFUNC_VX_IMPL is introduced and is used to register all ___c() and ___vx() functions within __libc_ifunc_impl_list() to the ifunc test framework. ChangeLog: * sysdeps/s390/multiarch/Makefile: New File. * sysdeps/s390/multiarch/strlen-c.c: Likewise. * sysdeps/s390/multiarch/strlen-vx.S: Likewise. * sysdeps/s390/multiarch/strlen.c: Likewise. * sysdeps/s390/multiarch/wcslen-c.c: Likewise. * sysdeps/s390/multiarch/wcslen-vx.S: Likewise. * sysdeps/s390/multiarch/wcslen.c: Likewise. * string/strlen.c (STRLEN): Define and use macro. * sysdeps/s390/multiarch/ifunc-impl-list.c (IFUNC_VX_IMPL): New macro function. (__libc_ifunc_impl_list): Add ifunc test for strlen, wcslen. * benchtests/Makefile (wcsmbs-bench): New variable. (string-bench-all): Added wcsmbs-bench. * benchtests/bench-wcslen.c: New File. --- benchtests/Makefile | 3 +- benchtests/bench-wcslen.c | 20 +++++++ string/strlen.c | 7 ++- sysdeps/s390/multiarch/Makefile | 7 +++ sysdeps/s390/multiarch/ifunc-impl-list.c | 14 +++++ sysdeps/s390/multiarch/strlen-c.c | 28 ++++++++++ sysdeps/s390/multiarch/strlen-vx.S | 84 +++++++++++++++++++++++++++++ sysdeps/s390/multiarch/strlen.c | 27 ++++++++++ sysdeps/s390/multiarch/wcslen-c.c | 25 +++++++++ sysdeps/s390/multiarch/wcslen-vx.S | 91 ++++++++++++++++++++++++++++++++ sysdeps/s390/multiarch/wcslen.c | 28 ++++++++++ 11 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 benchtests/bench-wcslen.c create mode 100644 sysdeps/s390/multiarch/Makefile create mode 100644 sysdeps/s390/multiarch/strlen-c.c create mode 100644 sysdeps/s390/multiarch/strlen-vx.S create mode 100644 sysdeps/s390/multiarch/strlen.c create mode 100644 sysdeps/s390/multiarch/wcslen-c.c create mode 100644 sysdeps/s390/multiarch/wcslen-vx.S create mode 100644 sysdeps/s390/multiarch/wcslen.c diff --git a/benchtests/Makefile b/benchtests/Makefile index 9b02bc7..09ab87f 100644 --- a/benchtests/Makefile +++ b/benchtests/Makefile @@ -38,7 +38,8 @@ string-bench := bcopy bzero memccpy memchr memcmp memcpy memmem memmove \ strcat strchr strchrnul strcmp strcpy strcspn strlen \ strncasecmp strncat strncmp strncpy strnlen strpbrk strrchr \ strspn strstr strcpy_chk stpcpy_chk memrchr strsep strtok -string-bench-all := $(string-bench) +wcsmbs-bench := wcslen +string-bench-all := $(string-bench) ${wcsmbs-bench} stdlib-bench := strtod diff --git a/benchtests/bench-wcslen.c b/benchtests/bench-wcslen.c new file mode 100644 index 0000000..4e9d085 --- /dev/null +++ b/benchtests/bench-wcslen.c @@ -0,0 +1,20 @@ +/* Measure wcslen functions. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#define WIDE 1 +#include "bench-strlen.c" diff --git a/string/strlen.c b/string/strlen.c index 5c1efda..d682693 100644 --- a/string/strlen.c +++ b/string/strlen.c @@ -23,11 +23,14 @@ #undef strlen +#ifndef STRLEN +# define STRLEN strlen +#endif + /* Return the length of the null-terminated string STR. Scan for the null terminator quickly by testing four bytes at a time. */ size_t -strlen (str) - const char *str; +STRLEN (const char *str) { const char *char_ptr; const unsigned long int *longword_ptr; diff --git a/sysdeps/s390/multiarch/Makefile b/sysdeps/s390/multiarch/Makefile new file mode 100644 index 0000000..3a98098 --- /dev/null +++ b/sysdeps/s390/multiarch/Makefile @@ -0,0 +1,7 @@ +ifeq ($(subdir),string) +sysdep_routines += strlen strlen-vx strlen-c +endif + +ifeq ($(subdir),wcsmbs) +sysdep_routines += wcslen wcslen-vx wcslen-c +endif diff --git a/sysdeps/s390/multiarch/ifunc-impl-list.c b/sysdeps/s390/multiarch/ifunc-impl-list.c index c330904..e9639ef 100644 --- a/sysdeps/s390/multiarch/ifunc-impl-list.c +++ b/sysdeps/s390/multiarch/ifunc-impl-list.c @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -70,5 +71,18 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, #endif /* SHARED */ +#ifdef HAVE_S390_VX_ASM_SUPPORT + +# define IFUNC_VX_IMPL(FUNC) \ + IFUNC_IMPL (i, name, FUNC, \ + IFUNC_IMPL_ADD (array, i, FUNC, dl_hwcap & HWCAP_S390_VX, \ + __##FUNC##_vx) \ + IFUNC_IMPL_ADD (array, i, FUNC, 1, __##FUNC##_c)) + + IFUNC_VX_IMPL (strlen); + IFUNC_VX_IMPL (wcslen); + +#endif /* HAVE_S390_VX_ASM_SUPPORT */ + return i; } diff --git a/sysdeps/s390/multiarch/strlen-c.c b/sysdeps/s390/multiarch/strlen-c.c new file mode 100644 index 0000000..1cbe959 --- /dev/null +++ b/sysdeps/s390/multiarch/strlen-c.c @@ -0,0 +1,28 @@ +/* Default strlen implementation for S/390. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc +# define STRLEN __strlen_c +# ifdef SHARED +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(name) \ + __hidden_ver1 (__strlen_c, __GI_strlen, __strlen_c); +# endif /* SHARED */ + +# include +#endif /* HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc */ diff --git a/sysdeps/s390/multiarch/strlen-vx.S b/sysdeps/s390/multiarch/strlen-vx.S new file mode 100644 index 0000000..1a5cb23 --- /dev/null +++ b/sysdeps/s390/multiarch/strlen-vx.S @@ -0,0 +1,84 @@ +/* Vector optimized 32/64 bit S/390 version of strlen. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc + +# include "sysdep.h" +# include "asm-syntax.h" + + .text + +/* size_t strlen (const char *s) + Returns length of string s. + + Register usage: + -r1=bytes to 4k-byte boundary + -r2=s + -r3=tmp + -r4=tmp + -r5=current_len and return_value + -v16=part of s +*/ +ENTRY(__strlen_vx) + .machine "z13" + .machinemode "zarch_nohighgprs" + + vlbb %v16,0(%r2),6 /* Load s until next 4k-byte boundary. */ + lcbb %r1,0(%r2),6 /* Get bytes to 4k-byte boundary or 16. */ + + vfenezb %v16,%v16,%v16 /* Find element not equal with zero search. */ + vlgvb %r4,%v16,7 /* Load zero index or 16 if not found. */ + clr %r4,%r1 /* If found zero within loaded bytes? */ + locgrl %r2,%r4 /* Then copy return value. */ + blr %r14 /* And return. */ + + /* Align s to 16 byte. */ + risbgn %r3,%r2,60,128+63,0 /* %r3 = bits 60-63 of %r2 'and' 15. */ + lghi %r5,16 /* current_len = 16. */ + slr %r5,%r3 /* Compute bytes to 16bytes boundary. */ + + /* Find zero in 16 byte aligned loop. */ +.Lloop: + vl %v16,0(%r5,%r2) /* Load s. */ + vfenezbs %v16,%v16,%v16 /* Find element not equal with zero search. */ + je .Lfound /* Jump away if zero was found. */ + vl %v16,16(%r5,%r2) + vfenezbs %v16,%v16,%v16 + je .Lfound16 + vl %v16,32(%r5,%r2) + vfenezbs %v16,%v16,%v16 + je .Lfound32 + vl %v16,48(%r5,%r2) + vfenezbs %v16,%v16,%v16 + je .Lfound48 + + aghi %r5,64 + j .Lloop /* No zero found -> loop. */ + +.Lfound48: + aghi %r5,16 +.Lfound32: + aghi %r5,16 +.Lfound16: + aghi %r5,16 +.Lfound: + vlgvb %r2,%v16,7 /* Load byte index of zero. */ + algr %r2,%r5 + br %r14 +END(__strlen_vx) +#endif /* HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc */ diff --git a/sysdeps/s390/multiarch/strlen.c b/sysdeps/s390/multiarch/strlen.c new file mode 100644 index 0000000..ba5863f --- /dev/null +++ b/sysdeps/s390/multiarch/strlen.c @@ -0,0 +1,27 @@ +/* Multiple versions of strlen. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc +# include +# include + +s390_vx_libc_ifunc2 (__strlen, strlen) + +#else +# include +#endif /* !(defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc) */ diff --git a/sysdeps/s390/multiarch/wcslen-c.c b/sysdeps/s390/multiarch/wcslen-c.c new file mode 100644 index 0000000..6dd011e --- /dev/null +++ b/sysdeps/s390/multiarch/wcslen-c.c @@ -0,0 +1,25 @@ +/* Default wcslen implementation for S/390. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc +# define WCSLEN __wcslen_c + +# include +extern __typeof (__wcslen) __wcslen_c; +# include +#endif diff --git a/sysdeps/s390/multiarch/wcslen-vx.S b/sysdeps/s390/multiarch/wcslen-vx.S new file mode 100644 index 0000000..579e66b --- /dev/null +++ b/sysdeps/s390/multiarch/wcslen-vx.S @@ -0,0 +1,91 @@ +/* Vector optimized 32/64 bit S/390 version of wcslen. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc + +# include "sysdep.h" +# include "asm-syntax.h" + + .text + +/* size_t wcslen (const wchar_t *s) + Returns length of string s. + + Register usage: + -r1=bytes to 4k-byte boundary + -r2=s + -r3=tmp + -r4=tmp + -r5=current_len and return_value + -v16=part of s +*/ +ENTRY(__wcslen_vx) + .machine "z13" + .machinemode "zarch_nohighgprs" + + vlbb %v16,0(%r2),6 /* Load s until next 4k-byte boundary. */ + lcbb %r1,0(%r2),6 /* Get bytes to 4k-byte boundary or 16. */ + + tmll %r2,3 /* Test if s is 4-byte aligned? */ + jne .Lfallback /* And use common-code variant if not. */ + + vfenezf %v16,%v16,%v16 /* Find element not equal with zero search. */ + vlgvb %r4,%v16,7 /* Load zero index or 16 if not found. */ + clr %r4,%r1 /* If found zero within loaded bytes? */ + locgrl %r2,%r4 /* Then copy return value. */ + jl .Lend /* And return. */ + + /* Align s to 16 byte. */ + risbgn %r3,%r2,60,128+63,0 /* %r3 = bits 60-63 of %r2 'and' 15. */ + lghi %r5,16 /* current_len = 16. */ + slr %r5,%r3 /* Compute bytes to 16bytes boundary. */ + + /* Find zero in 16byte aligned loop. */ +.Lloop: + vl %v16,0(%r5,%r2) /* Load s. */ + vfenezfs %v16,%v16,%v16 /* Find element not equal with zero search. */ + je .Lfound /* Jump away if zero was found. */ + vl %v16,16(%r5,%r2) + vfenezfs %v16,%v16,%v16 + je .Lfound16 + vl %v16,32(%r5,%r2) + vfenezfs %v16,%v16,%v16 + je .Lfound32 + vl %v16,48(%r5,%r2) + vfenezfs %v16,%v16,%v16 + je .Lfound48 + + aghi %r5,64 + j .Lloop /* No zero found -> loop. */ + +.Lfound48: + aghi %r5,16 +.Lfound32: + aghi %r5,16 +.Lfound16: + aghi %r5,16 +.Lfound: + vlgvb %r2,%v16,7 /* Load byte index of zero. */ + algr %r2,%r5 +.Lend: + srlg %r2,%r2,2 /* Convert byte-count to character-count. */ + br %r14 +.Lfallback: + jg __wcslen_c +END(__wcslen_vx) +#endif /* HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc */ diff --git a/sysdeps/s390/multiarch/wcslen.c b/sysdeps/s390/multiarch/wcslen.c new file mode 100644 index 0000000..a7be73e --- /dev/null +++ b/sysdeps/s390/multiarch/wcslen.c @@ -0,0 +1,28 @@ +/* Multiple versions of wcslen. + Copyright (C) 2015 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + . */ + +#if defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc +# include +# include + +s390_vx_libc_ifunc (__wcslen) +weak_alias (__wcslen, wcslen) + +#else +# include +#endif /* !(defined HAVE_S390_VX_ASM_SUPPORT && !defined NOT_IN_libc) */ -- 2.3.0