You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
338 lines
8.9 KiB
338 lines
8.9 KiB
commit dd6d3a0bbcc67cb2b50b0add0c599f9f99491d8b |
|
Author: Noah Goldstein <goldstein.w.n@gmail.com> |
|
Date: Wed Mar 23 16:57:18 2022 -0500 |
|
|
|
x86: Code cleanup in strchr-evex and comment justifying branch |
|
|
|
Small code cleanup for size: -81 bytes. |
|
|
|
Add comment justifying using a branch to do NULL/non-null return. |
|
|
|
All string/memory tests pass and no regressions in benchtests. |
|
|
|
geometric_mean(N=20) of all benchmarks New / Original: .985 |
|
Reviewed-by: H.J. Lu <hjl.tools@gmail.com> |
|
|
|
(cherry picked from commit ec285ea90415458225623ddc0492ae3f705af043) |
|
|
|
diff --git a/sysdeps/x86_64/multiarch/strchr-evex.S b/sysdeps/x86_64/multiarch/strchr-evex.S |
|
index 7f9d4ee48ddaa998..0b49e0ac54e7b0dd 100644 |
|
--- a/sysdeps/x86_64/multiarch/strchr-evex.S |
|
+++ b/sysdeps/x86_64/multiarch/strchr-evex.S |
|
@@ -30,6 +30,7 @@ |
|
# ifdef USE_AS_WCSCHR |
|
# define VPBROADCAST vpbroadcastd |
|
# define VPCMP vpcmpd |
|
+# define VPTESTN vptestnmd |
|
# define VPMINU vpminud |
|
# define CHAR_REG esi |
|
# define SHIFT_REG ecx |
|
@@ -37,6 +38,7 @@ |
|
# else |
|
# define VPBROADCAST vpbroadcastb |
|
# define VPCMP vpcmpb |
|
+# define VPTESTN vptestnmb |
|
# define VPMINU vpminub |
|
# define CHAR_REG sil |
|
# define SHIFT_REG edx |
|
@@ -61,13 +63,11 @@ |
|
# define CHAR_PER_VEC (VEC_SIZE / CHAR_SIZE) |
|
|
|
.section .text.evex,"ax",@progbits |
|
-ENTRY (STRCHR) |
|
+ENTRY_P2ALIGN (STRCHR, 5) |
|
/* Broadcast CHAR to YMM0. */ |
|
VPBROADCAST %esi, %YMM0 |
|
movl %edi, %eax |
|
andl $(PAGE_SIZE - 1), %eax |
|
- vpxorq %XMMZERO, %XMMZERO, %XMMZERO |
|
- |
|
/* Check if we cross page boundary with one vector load. |
|
Otherwise it is safe to use an unaligned load. */ |
|
cmpl $(PAGE_SIZE - VEC_SIZE), %eax |
|
@@ -81,49 +81,35 @@ ENTRY (STRCHR) |
|
vpxorq %YMM1, %YMM0, %YMM2 |
|
VPMINU %YMM2, %YMM1, %YMM2 |
|
/* Each bit in K0 represents a CHAR or a null byte in YMM1. */ |
|
- VPCMP $0, %YMMZERO, %YMM2, %k0 |
|
+ VPTESTN %YMM2, %YMM2, %k0 |
|
kmovd %k0, %eax |
|
testl %eax, %eax |
|
jz L(aligned_more) |
|
tzcntl %eax, %eax |
|
+# ifndef USE_AS_STRCHRNUL |
|
+ /* Found CHAR or the null byte. */ |
|
+ cmp (%rdi, %rax, CHAR_SIZE), %CHAR_REG |
|
+ /* NB: Use a branch instead of cmovcc here. The expectation is |
|
+ that with strchr the user will branch based on input being |
|
+ null. Since this branch will be 100% predictive of the user |
|
+ branch a branch miss here should save what otherwise would |
|
+ be branch miss in the user code. Otherwise using a branch 1) |
|
+ saves code size and 2) is faster in highly predictable |
|
+ environments. */ |
|
+ jne L(zero) |
|
+# endif |
|
# ifdef USE_AS_WCSCHR |
|
/* NB: Multiply wchar_t count by 4 to get the number of bytes. |
|
*/ |
|
leaq (%rdi, %rax, CHAR_SIZE), %rax |
|
# else |
|
addq %rdi, %rax |
|
-# endif |
|
-# ifndef USE_AS_STRCHRNUL |
|
- /* Found CHAR or the null byte. */ |
|
- cmp (%rax), %CHAR_REG |
|
- jne L(zero) |
|
# endif |
|
ret |
|
|
|
- /* .p2align 5 helps keep performance more consistent if ENTRY() |
|
- alignment % 32 was either 16 or 0. As well this makes the |
|
- alignment % 32 of the loop_4x_vec fixed which makes tuning it |
|
- easier. */ |
|
- .p2align 5 |
|
-L(first_vec_x3): |
|
- tzcntl %eax, %eax |
|
-# ifndef USE_AS_STRCHRNUL |
|
- /* Found CHAR or the null byte. */ |
|
- cmp (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG |
|
- jne L(zero) |
|
-# endif |
|
- /* NB: Multiply sizeof char type (1 or 4) to get the number of |
|
- bytes. */ |
|
- leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax |
|
- ret |
|
|
|
-# ifndef USE_AS_STRCHRNUL |
|
-L(zero): |
|
- xorl %eax, %eax |
|
- ret |
|
-# endif |
|
|
|
- .p2align 4 |
|
+ .p2align 4,, 10 |
|
L(first_vec_x4): |
|
# ifndef USE_AS_STRCHRNUL |
|
/* Check to see if first match was CHAR (k0) or null (k1). */ |
|
@@ -144,9 +130,18 @@ L(first_vec_x4): |
|
leaq (VEC_SIZE * 4)(%rdi, %rax, CHAR_SIZE), %rax |
|
ret |
|
|
|
+# ifndef USE_AS_STRCHRNUL |
|
+L(zero): |
|
+ xorl %eax, %eax |
|
+ ret |
|
+# endif |
|
+ |
|
+ |
|
.p2align 4 |
|
L(first_vec_x1): |
|
- tzcntl %eax, %eax |
|
+ /* Use bsf here to save 1-byte keeping keeping the block in 1x |
|
+ fetch block. eax guranteed non-zero. */ |
|
+ bsfl %eax, %eax |
|
# ifndef USE_AS_STRCHRNUL |
|
/* Found CHAR or the null byte. */ |
|
cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG |
|
@@ -158,7 +153,7 @@ L(first_vec_x1): |
|
leaq (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %rax |
|
ret |
|
|
|
- .p2align 4 |
|
+ .p2align 4,, 10 |
|
L(first_vec_x2): |
|
# ifndef USE_AS_STRCHRNUL |
|
/* Check to see if first match was CHAR (k0) or null (k1). */ |
|
@@ -179,6 +174,21 @@ L(first_vec_x2): |
|
leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax |
|
ret |
|
|
|
+ .p2align 4,, 10 |
|
+L(first_vec_x3): |
|
+ /* Use bsf here to save 1-byte keeping keeping the block in 1x |
|
+ fetch block. eax guranteed non-zero. */ |
|
+ bsfl %eax, %eax |
|
+# ifndef USE_AS_STRCHRNUL |
|
+ /* Found CHAR or the null byte. */ |
|
+ cmp (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %CHAR_REG |
|
+ jne L(zero) |
|
+# endif |
|
+ /* NB: Multiply sizeof char type (1 or 4) to get the number of |
|
+ bytes. */ |
|
+ leaq (VEC_SIZE * 3)(%rdi, %rax, CHAR_SIZE), %rax |
|
+ ret |
|
+ |
|
.p2align 4 |
|
L(aligned_more): |
|
/* Align data to VEC_SIZE. */ |
|
@@ -195,7 +205,7 @@ L(cross_page_continue): |
|
vpxorq %YMM1, %YMM0, %YMM2 |
|
VPMINU %YMM2, %YMM1, %YMM2 |
|
/* Each bit in K0 represents a CHAR or a null byte in YMM1. */ |
|
- VPCMP $0, %YMMZERO, %YMM2, %k0 |
|
+ VPTESTN %YMM2, %YMM2, %k0 |
|
kmovd %k0, %eax |
|
testl %eax, %eax |
|
jnz L(first_vec_x1) |
|
@@ -206,7 +216,7 @@ L(cross_page_continue): |
|
/* Each bit in K0 represents a CHAR in YMM1. */ |
|
VPCMP $0, %YMM1, %YMM0, %k0 |
|
/* Each bit in K1 represents a CHAR in YMM1. */ |
|
- VPCMP $0, %YMM1, %YMMZERO, %k1 |
|
+ VPTESTN %YMM1, %YMM1, %k1 |
|
kortestd %k0, %k1 |
|
jnz L(first_vec_x2) |
|
|
|
@@ -215,7 +225,7 @@ L(cross_page_continue): |
|
vpxorq %YMM1, %YMM0, %YMM2 |
|
VPMINU %YMM2, %YMM1, %YMM2 |
|
/* Each bit in K0 represents a CHAR or a null byte in YMM1. */ |
|
- VPCMP $0, %YMMZERO, %YMM2, %k0 |
|
+ VPTESTN %YMM2, %YMM2, %k0 |
|
kmovd %k0, %eax |
|
testl %eax, %eax |
|
jnz L(first_vec_x3) |
|
@@ -224,7 +234,7 @@ L(cross_page_continue): |
|
/* Each bit in K0 represents a CHAR in YMM1. */ |
|
VPCMP $0, %YMM1, %YMM0, %k0 |
|
/* Each bit in K1 represents a CHAR in YMM1. */ |
|
- VPCMP $0, %YMM1, %YMMZERO, %k1 |
|
+ VPTESTN %YMM1, %YMM1, %k1 |
|
kortestd %k0, %k1 |
|
jnz L(first_vec_x4) |
|
|
|
@@ -265,33 +275,33 @@ L(loop_4x_vec): |
|
VPMINU %YMM3, %YMM4, %YMM4 |
|
VPMINU %YMM2, %YMM4, %YMM4{%k4}{z} |
|
|
|
- VPCMP $0, %YMMZERO, %YMM4, %k1 |
|
+ VPTESTN %YMM4, %YMM4, %k1 |
|
kmovd %k1, %ecx |
|
subq $-(VEC_SIZE * 4), %rdi |
|
testl %ecx, %ecx |
|
jz L(loop_4x_vec) |
|
|
|
- VPCMP $0, %YMMZERO, %YMM1, %k0 |
|
+ VPTESTN %YMM1, %YMM1, %k0 |
|
kmovd %k0, %eax |
|
testl %eax, %eax |
|
jnz L(last_vec_x1) |
|
|
|
- VPCMP $0, %YMMZERO, %YMM2, %k0 |
|
+ VPTESTN %YMM2, %YMM2, %k0 |
|
kmovd %k0, %eax |
|
testl %eax, %eax |
|
jnz L(last_vec_x2) |
|
|
|
- VPCMP $0, %YMMZERO, %YMM3, %k0 |
|
+ VPTESTN %YMM3, %YMM3, %k0 |
|
kmovd %k0, %eax |
|
/* Combine YMM3 matches (eax) with YMM4 matches (ecx). */ |
|
# ifdef USE_AS_WCSCHR |
|
sall $8, %ecx |
|
orl %ecx, %eax |
|
- tzcntl %eax, %eax |
|
+ bsfl %eax, %eax |
|
# else |
|
salq $32, %rcx |
|
orq %rcx, %rax |
|
- tzcntq %rax, %rax |
|
+ bsfq %rax, %rax |
|
# endif |
|
# ifndef USE_AS_STRCHRNUL |
|
/* Check if match was CHAR or null. */ |
|
@@ -303,28 +313,28 @@ L(loop_4x_vec): |
|
leaq (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %rax |
|
ret |
|
|
|
-# ifndef USE_AS_STRCHRNUL |
|
-L(zero_end): |
|
- xorl %eax, %eax |
|
- ret |
|
+ .p2align 4,, 8 |
|
+L(last_vec_x1): |
|
+ bsfl %eax, %eax |
|
+# ifdef USE_AS_WCSCHR |
|
+ /* NB: Multiply wchar_t count by 4 to get the number of bytes. |
|
+ */ |
|
+ leaq (%rdi, %rax, CHAR_SIZE), %rax |
|
+# else |
|
+ addq %rdi, %rax |
|
# endif |
|
|
|
- .p2align 4 |
|
-L(last_vec_x1): |
|
- tzcntl %eax, %eax |
|
# ifndef USE_AS_STRCHRNUL |
|
/* Check if match was null. */ |
|
- cmp (%rdi, %rax, CHAR_SIZE), %CHAR_REG |
|
+ cmp (%rax), %CHAR_REG |
|
jne L(zero_end) |
|
# endif |
|
- /* NB: Multiply sizeof char type (1 or 4) to get the number of |
|
- bytes. */ |
|
- leaq (%rdi, %rax, CHAR_SIZE), %rax |
|
+ |
|
ret |
|
|
|
- .p2align 4 |
|
+ .p2align 4,, 8 |
|
L(last_vec_x2): |
|
- tzcntl %eax, %eax |
|
+ bsfl %eax, %eax |
|
# ifndef USE_AS_STRCHRNUL |
|
/* Check if match was null. */ |
|
cmp (VEC_SIZE)(%rdi, %rax, CHAR_SIZE), %CHAR_REG |
|
@@ -336,7 +346,7 @@ L(last_vec_x2): |
|
ret |
|
|
|
/* Cold case for crossing page with first load. */ |
|
- .p2align 4 |
|
+ .p2align 4,, 8 |
|
L(cross_page_boundary): |
|
movq %rdi, %rdx |
|
/* Align rdi. */ |
|
@@ -346,9 +356,9 @@ L(cross_page_boundary): |
|
vpxorq %YMM1, %YMM0, %YMM2 |
|
VPMINU %YMM2, %YMM1, %YMM2 |
|
/* Each bit in K0 represents a CHAR or a null byte in YMM1. */ |
|
- VPCMP $0, %YMMZERO, %YMM2, %k0 |
|
+ VPTESTN %YMM2, %YMM2, %k0 |
|
kmovd %k0, %eax |
|
- /* Remove the leading bits. */ |
|
+ /* Remove the leading bits. */ |
|
# ifdef USE_AS_WCSCHR |
|
movl %edx, %SHIFT_REG |
|
/* NB: Divide shift count by 4 since each bit in K1 represent 4 |
|
@@ -360,20 +370,24 @@ L(cross_page_boundary): |
|
/* If eax is zero continue. */ |
|
testl %eax, %eax |
|
jz L(cross_page_continue) |
|
- tzcntl %eax, %eax |
|
-# ifndef USE_AS_STRCHRNUL |
|
- /* Check to see if match was CHAR or null. */ |
|
- cmp (%rdx, %rax, CHAR_SIZE), %CHAR_REG |
|
- jne L(zero_end) |
|
-# endif |
|
+ bsfl %eax, %eax |
|
+ |
|
# ifdef USE_AS_WCSCHR |
|
/* NB: Multiply wchar_t count by 4 to get the number of |
|
bytes. */ |
|
leaq (%rdx, %rax, CHAR_SIZE), %rax |
|
# else |
|
addq %rdx, %rax |
|
+# endif |
|
+# ifndef USE_AS_STRCHRNUL |
|
+ /* Check to see if match was CHAR or null. */ |
|
+ cmp (%rax), %CHAR_REG |
|
+ je L(cross_page_ret) |
|
+L(zero_end): |
|
+ xorl %eax, %eax |
|
+L(cross_page_ret): |
|
# endif |
|
ret |
|
|
|
END (STRCHR) |
|
-# endif |
|
+#endif
|
|
|