You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
323 lines
8.4 KiB
323 lines
8.4 KiB
commit a910d7e164f1d9b8e77bbea35a2d2ab89a5e26cc |
|
Author: Noah Goldstein <goldstein.w.n@gmail.com> |
|
Date: Mon Jun 6 21:11:33 2022 -0700 |
|
|
|
x86: Shrink code size of memchr-avx2.S |
|
|
|
This is not meant as a performance optimization. The previous code was |
|
far to liberal in aligning targets and wasted code size unnecissarily. |
|
|
|
The total code size saving is: 59 bytes |
|
|
|
There are no major changes in the benchmarks. |
|
Geometric Mean of all benchmarks New / Old: 0.967 |
|
|
|
Full xcheck passes on x86_64. |
|
Reviewed-by: H.J. Lu <hjl.tools@gmail.com> |
|
|
|
(cherry picked from commit 6dcbb7d95dded20153b12d76d2f4e0ef0cda4f35) |
|
|
|
x86: Fix page cross case in rawmemchr-avx2 [BZ #29234] |
|
|
|
commit 6dcbb7d95dded20153b12d76d2f4e0ef0cda4f35 |
|
Author: Noah Goldstein <goldstein.w.n@gmail.com> |
|
Date: Mon Jun 6 21:11:33 2022 -0700 |
|
|
|
x86: Shrink code size of memchr-avx2.S |
|
|
|
Changed how the page cross case aligned string (rdi) in |
|
rawmemchr. This was incompatible with how |
|
`L(cross_page_continue)` expected the pointer to be aligned and |
|
would cause rawmemchr to read data start started before the |
|
beginning of the string. What it would read was in valid memory |
|
but could count CHAR matches resulting in an incorrect return |
|
value. |
|
|
|
This commit fixes that issue by essentially reverting the changes to |
|
the L(page_cross) case as they didn't really matter. |
|
|
|
Test cases added and all pass with the new code (and where confirmed |
|
to fail with the old code). |
|
Reviewed-by: H.J. Lu <hjl.tools@gmail.com> |
|
|
|
(cherry picked from commit 2c9af8421d2b4a7fcce163e7bc81a118d22fd346) |
|
|
|
diff --git a/string/test-rawmemchr.c b/string/test-rawmemchr.c |
|
index 085098aba8fdbc13..327c0654e69e7669 100644 |
|
--- a/string/test-rawmemchr.c |
|
+++ b/string/test-rawmemchr.c |
|
@@ -18,6 +18,7 @@ |
|
<https://www.gnu.org/licenses/>. */ |
|
|
|
#include <assert.h> |
|
+#include <support/xunistd.h> |
|
|
|
#define TEST_MAIN |
|
#define TEST_NAME "rawmemchr" |
|
@@ -51,13 +52,45 @@ do_one_test (impl_t *impl, const char *s, int c, char *exp_res) |
|
} |
|
} |
|
|
|
+static void |
|
+do_test_bz29234 (void) |
|
+{ |
|
+ size_t i, j; |
|
+ char *ptr_start; |
|
+ char *buf = xmmap (0, 8192, PROT_READ | PROT_WRITE, |
|
+ MAP_PRIVATE | MAP_ANONYMOUS, -1); |
|
+ |
|
+ memset (buf, -1, 8192); |
|
+ |
|
+ ptr_start = buf + 4096 - 8; |
|
+ |
|
+ /* Out of range matches before the start of a page. */ |
|
+ memset (ptr_start - 8, 0x1, 8); |
|
+ |
|
+ for (j = 0; j < 8; ++j) |
|
+ { |
|
+ for (i = 0; i < 128; ++i) |
|
+ { |
|
+ ptr_start[i + j] = 0x1; |
|
+ |
|
+ FOR_EACH_IMPL (impl, 0) |
|
+ do_one_test (impl, (char *) (ptr_start + j), 0x1, |
|
+ ptr_start + i + j); |
|
+ |
|
+ ptr_start[i + j] = 0xff; |
|
+ } |
|
+ } |
|
+ |
|
+ xmunmap (buf, 8192); |
|
+} |
|
+ |
|
static void |
|
do_test (size_t align, size_t pos, size_t len, int seek_char) |
|
{ |
|
size_t i; |
|
char *result; |
|
|
|
- align &= 7; |
|
+ align &= getpagesize () - 1; |
|
if (align + len >= page_size) |
|
return; |
|
|
|
@@ -115,6 +148,13 @@ do_random_tests (void) |
|
} |
|
} |
|
|
|
+ if (align) |
|
+ { |
|
+ p[align - 1] = seek_char; |
|
+ if (align > 4) |
|
+ p[align - 4] = seek_char; |
|
+ } |
|
+ |
|
assert (pos < len); |
|
size_t r = random (); |
|
if ((r & 31) == 0) |
|
@@ -130,6 +170,13 @@ do_random_tests (void) |
|
result, p); |
|
ret = 1; |
|
} |
|
+ |
|
+ if (align) |
|
+ { |
|
+ p[align - 1] = seek_char; |
|
+ if (align > 4) |
|
+ p[align - 4] = seek_char; |
|
+ } |
|
} |
|
} |
|
|
|
@@ -151,14 +198,22 @@ test_main (void) |
|
do_test (i, 64, 256, 23); |
|
do_test (0, 16 << i, 2048, 0); |
|
do_test (i, 64, 256, 0); |
|
+ |
|
+ do_test (getpagesize () - i, 64, 256, 23); |
|
+ do_test (getpagesize () - i, 64, 256, 0); |
|
} |
|
for (i = 1; i < 32; ++i) |
|
{ |
|
do_test (0, i, i + 1, 23); |
|
do_test (0, i, i + 1, 0); |
|
+ |
|
+ do_test (getpagesize () - 7, i, i + 1, 23); |
|
+ do_test (getpagesize () - i / 2, i, i + 1, 23); |
|
+ do_test (getpagesize () - i, i, i + 1, 23); |
|
} |
|
|
|
do_random_tests (); |
|
+ do_test_bz29234 (); |
|
return ret; |
|
} |
|
|
|
diff --git a/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S |
|
index 87b076c7c403ba85..c4d71938c5a3ed24 100644 |
|
--- a/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S |
|
+++ b/sysdeps/x86_64/multiarch/memchr-avx2-rtm.S |
|
@@ -2,6 +2,7 @@ |
|
# define MEMCHR __memchr_avx2_rtm |
|
#endif |
|
|
|
+#define COND_VZEROUPPER COND_VZEROUPPER_XTEST |
|
#define ZERO_UPPER_VEC_REGISTERS_RETURN \ |
|
ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST |
|
|
|
diff --git a/sysdeps/x86_64/multiarch/memchr-avx2.S b/sysdeps/x86_64/multiarch/memchr-avx2.S |
|
index afdb95650232fdac..9e0b7dd1f4fe9909 100644 |
|
--- a/sysdeps/x86_64/multiarch/memchr-avx2.S |
|
+++ b/sysdeps/x86_64/multiarch/memchr-avx2.S |
|
@@ -57,7 +57,7 @@ |
|
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) |
|
|
|
.section SECTION(.text),"ax",@progbits |
|
-ENTRY (MEMCHR) |
|
+ENTRY_P2ALIGN (MEMCHR, 5) |
|
# ifndef USE_AS_RAWMEMCHR |
|
/* Check for zero length. */ |
|
# ifdef __ILP32__ |
|
@@ -87,12 +87,14 @@ ENTRY (MEMCHR) |
|
# endif |
|
testl %eax, %eax |
|
jz L(aligned_more) |
|
- tzcntl %eax, %eax |
|
+ bsfl %eax, %eax |
|
addq %rdi, %rax |
|
- VZEROUPPER_RETURN |
|
+L(return_vzeroupper): |
|
+ ZERO_UPPER_VEC_REGISTERS_RETURN |
|
+ |
|
|
|
# ifndef USE_AS_RAWMEMCHR |
|
- .p2align 5 |
|
+ .p2align 4 |
|
L(first_vec_x0): |
|
/* Check if first match was before length. */ |
|
tzcntl %eax, %eax |
|
@@ -100,58 +102,31 @@ L(first_vec_x0): |
|
/* NB: Multiply length by 4 to get byte count. */ |
|
sall $2, %edx |
|
# endif |
|
- xorl %ecx, %ecx |
|
+ COND_VZEROUPPER |
|
+ /* Use branch instead of cmovcc so L(first_vec_x0) fits in one fetch |
|
+ block. branch here as opposed to cmovcc is not that costly. Common |
|
+ usage of memchr is to check if the return was NULL (if string was |
|
+ known to contain CHAR user would use rawmemchr). This branch will be |
|
+ highly correlated with the user branch and can be used by most |
|
+ modern branch predictors to predict the user branch. */ |
|
cmpl %eax, %edx |
|
- leaq (%rdi, %rax), %rax |
|
- cmovle %rcx, %rax |
|
- VZEROUPPER_RETURN |
|
- |
|
-L(null): |
|
- xorl %eax, %eax |
|
- ret |
|
-# endif |
|
- .p2align 4 |
|
-L(cross_page_boundary): |
|
- /* Save pointer before aligning as its original value is |
|
- necessary for computer return address if byte is found or |
|
- adjusting length if it is not and this is memchr. */ |
|
- movq %rdi, %rcx |
|
- /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr |
|
- and rdi for rawmemchr. */ |
|
- orq $(VEC_SIZE - 1), %ALGN_PTR_REG |
|
- VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1 |
|
- vpmovmskb %ymm1, %eax |
|
-# ifndef USE_AS_RAWMEMCHR |
|
- /* Calculate length until end of page (length checked for a |
|
- match). */ |
|
- leaq 1(%ALGN_PTR_REG), %rsi |
|
- subq %RRAW_PTR_REG, %rsi |
|
-# ifdef USE_AS_WMEMCHR |
|
- /* NB: Divide bytes by 4 to get wchar_t count. */ |
|
- shrl $2, %esi |
|
-# endif |
|
-# endif |
|
- /* Remove the leading bytes. */ |
|
- sarxl %ERAW_PTR_REG, %eax, %eax |
|
-# ifndef USE_AS_RAWMEMCHR |
|
- /* Check the end of data. */ |
|
- cmpq %rsi, %rdx |
|
- jbe L(first_vec_x0) |
|
+ jle L(null) |
|
+ addq %rdi, %rax |
|
+ ret |
|
# endif |
|
- testl %eax, %eax |
|
- jz L(cross_page_continue) |
|
- tzcntl %eax, %eax |
|
- addq %RRAW_PTR_REG, %rax |
|
-L(return_vzeroupper): |
|
- ZERO_UPPER_VEC_REGISTERS_RETURN |
|
|
|
- .p2align 4 |
|
+ .p2align 4,, 10 |
|
L(first_vec_x1): |
|
- tzcntl %eax, %eax |
|
+ bsfl %eax, %eax |
|
incq %rdi |
|
addq %rdi, %rax |
|
VZEROUPPER_RETURN |
|
- |
|
+# ifndef USE_AS_RAWMEMCHR |
|
+ /* First in aligning bytes here. */ |
|
+L(null): |
|
+ xorl %eax, %eax |
|
+ ret |
|
+# endif |
|
.p2align 4 |
|
L(first_vec_x2): |
|
tzcntl %eax, %eax |
|
@@ -340,7 +315,7 @@ L(first_vec_x1_check): |
|
incq %rdi |
|
addq %rdi, %rax |
|
VZEROUPPER_RETURN |
|
- .p2align 4 |
|
+ .p2align 4,, 6 |
|
L(set_zero_end): |
|
xorl %eax, %eax |
|
VZEROUPPER_RETURN |
|
@@ -428,5 +403,39 @@ L(last_vec_x3): |
|
VZEROUPPER_RETURN |
|
# endif |
|
|
|
+ .p2align 4 |
|
+L(cross_page_boundary): |
|
+ /* Save pointer before aligning as its original value is necessary for |
|
+ computer return address if byte is found or adjusting length if it |
|
+ is not and this is memchr. */ |
|
+ movq %rdi, %rcx |
|
+ /* Align data to VEC_SIZE - 1. ALGN_PTR_REG is rcx for memchr |
|
+ and rdi for rawmemchr. */ |
|
+ orq $(VEC_SIZE - 1), %ALGN_PTR_REG |
|
+ VPCMPEQ -(VEC_SIZE - 1)(%ALGN_PTR_REG), %ymm0, %ymm1 |
|
+ vpmovmskb %ymm1, %eax |
|
+# ifndef USE_AS_RAWMEMCHR |
|
+ /* Calculate length until end of page (length checked for a match). */ |
|
+ leaq 1(%ALGN_PTR_REG), %rsi |
|
+ subq %RRAW_PTR_REG, %rsi |
|
+# ifdef USE_AS_WMEMCHR |
|
+ /* NB: Divide bytes by 4 to get wchar_t count. */ |
|
+ shrl $2, %esi |
|
+# endif |
|
+# endif |
|
+ /* Remove the leading bytes. */ |
|
+ sarxl %ERAW_PTR_REG, %eax, %eax |
|
+# ifndef USE_AS_RAWMEMCHR |
|
+ /* Check the end of data. */ |
|
+ cmpq %rsi, %rdx |
|
+ jbe L(first_vec_x0) |
|
+# endif |
|
+ testl %eax, %eax |
|
+ jz L(cross_page_continue) |
|
+ bsfl %eax, %eax |
|
+ addq %RRAW_PTR_REG, %rax |
|
+ VZEROUPPER_RETURN |
|
+ |
|
+ |
|
END (MEMCHR) |
|
#endif
|
|
|