Repository navigation
Expand file tree
/
Copy pathsimd-checksum-avx2.S
More file actions
207 lines (192 loc) · 4.34 KB
/
Copy pathsimd-checksum-avx2.S
File metadata and controls
207 lines (192 loc) · 4.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
#include "config.h"
#ifdef USE_ROLL_ASM /* { */
#define CHAR_OFFSET 0 /* Keep this the same as rsync.h, which isn't likely to change. */
#ifdef __APPLE__
#define get_checksum1_avx2_asm _get_checksum1_avx2_asm
#endif
.intel_syntax noprefix
.text
.p2align 5
.globl get_checksum1_avx2_asm
# rdi=*buf, esi=len, edx=i, rcx= *ps1, r8= *ps2
get_checksum1_avx2_asm:
vmovd xmm6,[rcx] # load *ps1
lea eax, [rsi-128] # at least 128 bytes to process?
cmp edx, eax
jg .exit
lea rax, .mul_T2[rip]
vmovntdqa ymm7, [rax] # load T2 multiplication constants
vmovntdqa ymm12,[rax+32]# from memory.
vpcmpeqd ymm15, ymm15, ymm15 # set all elements to -1.
#if CHAR_OFFSET != 0
mov eax, 32*CHAR_OFFSET
vmovd xmm10, eax
vpbroadcastd ymm10, xmm10
mov eax, 528*CHAR_OFFSET
vmovd xmm13, eax
vpbroadcastd ymm13, xmm13
#endif
vpabsb ymm15, ymm15 # set all byte size elements to 1.
add rdi, rdx
vmovdqu ymm2, [rdi] # preload the first 64 bytes.
vmovdqu ymm3, [rdi+32]
and esi, ~63 # only needed during final reduction,
# done here to avoid a longer nop for
# alignment below.
add edx, esi
shr rsi, 6 # longer opcode for alignment
add rdi, 64
vpxor xmm1, xmm1, xmm1 # reset both partial sums accumulators.
vpxor xmm4, xmm4, xmm4
mov eax, [r8]
# The loop below is software-pipelined: it preloads the 64 bytes AFTER the
# ones it is folding in. Run it one block short and finish that block in
# .last, which does not preload -- otherwise the final iteration reads 64
# bytes past the end of what the caller gave us. The remainder (len & 63)
# is always under 64, so that read always crossed buf+len; it only faulted
# when the buffer happened to end at a page boundary.
sub esi, 1
jz .last
.p2align 4 # should fit into the LSD allocation queue.
.loop:
vpmaddubsw ymm0, ymm15, ymm2 # s1 partial sums
vpmaddubsw ymm5, ymm15, ymm3
vmovdqu ymm8, [rdi] # preload the next
vmovdqu ymm9, [rdi+32] # 64 bytes.
add rdi, 64
vpaddd ymm4, ymm4, ymm6
vpaddw ymm5, ymm5, ymm0
vpsrld ymm0, ymm5, 16
vpaddw ymm5, ymm0, ymm5
vpaddd ymm6, ymm5, ymm6
vpmaddubsw ymm2, ymm7, ymm2 # s2 partial sums
vpmaddubsw ymm3, ymm12, ymm3
prefetcht0 [rdi+384] # prefetch 6 cachelines ahead.
vpaddw ymm3, ymm2, ymm3
vpsrldq ymm2, ymm3, 2
vpaddd ymm3, ymm2, ymm3
vpaddd ymm1, ymm1, ymm3
#if CHAR_OFFSET != 0
vpaddd ymm6, ymm10, ymm6 # 32*CHAR_OFFSET
vpaddd ymm1, ymm13, ymm1 # 528*CHAR_OFFSET
#endif
vmovdqa ymm2, ymm8 # move the next 64 bytes
vmovdqa ymm3, ymm9 # into the right registers
sub esi, 1
jnz .loop
# The final 64 bytes, already in ymm2/ymm3. Same arithmetic as the loop
# body with the preload, the prefetch and the loop control dropped.
.last:
vpmaddubsw ymm0, ymm15, ymm2 # s1 partial sums
vpmaddubsw ymm5, ymm15, ymm3
vpaddd ymm4, ymm4, ymm6
vpaddw ymm5, ymm5, ymm0
vpsrld ymm0, ymm5, 16
vpaddw ymm5, ymm0, ymm5
vpaddd ymm6, ymm5, ymm6
vpmaddubsw ymm2, ymm7, ymm2 # s2 partial sums
vpmaddubsw ymm3, ymm12, ymm3
vpaddw ymm3, ymm2, ymm3
vpsrldq ymm2, ymm3, 2
vpaddd ymm3, ymm2, ymm3
vpaddd ymm1, ymm1, ymm3
#if CHAR_OFFSET != 0
vpaddd ymm6, ymm10, ymm6 # 32*CHAR_OFFSET
vpaddd ymm1, ymm13, ymm1 # 528*CHAR_OFFSET
#endif
# now we reduce the partial sums.
vpslld ymm3, ymm4, 6
vpsrldq ymm2, ymm6, 4
vpaddd ymm0, ymm3, ymm1
vpaddd ymm6, ymm2, ymm6
vpsrlq ymm3, ymm0, 32
vpsrldq ymm2, ymm6, 8
vpaddd ymm0, ymm3, ymm0
vpsrldq ymm3, ymm0, 8
vpaddd ymm6, ymm2, ymm6
vpaddd ymm0, ymm3, ymm0
vextracti128 xmm2, ymm6, 0x1
vextracti128 xmm1, ymm0, 0x1
vpaddd xmm6, xmm2, xmm6
vmovd [rcx], xmm6
vpaddd xmm1, xmm1, xmm0
vmovd ecx, xmm1
add eax, ecx
mov [r8], eax
.exit:
vzeroupper
mov eax, edx
ret
#ifdef __APPLE__
.data
.align 6
#else
.section .rodata
.p2align 6
#endif
.mul_T2:
.byte 64
.byte 63
.byte 62
.byte 61
.byte 60
.byte 59
.byte 58
.byte 57
.byte 56
.byte 55
.byte 54
.byte 53
.byte 52
.byte 51
.byte 50
.byte 49
.byte 48
.byte 47
.byte 46
.byte 45
.byte 44
.byte 43
.byte 42
.byte 41
.byte 40
.byte 39
.byte 38
.byte 37
.byte 36
.byte 35
.byte 34
.byte 33
.byte 32
.byte 31
.byte 30
.byte 29
.byte 28
.byte 27
.byte 26
.byte 25
.byte 24
.byte 23
.byte 22
.byte 21
.byte 20
.byte 19
.byte 18
.byte 17
.byte 16
.byte 15
.byte 14
.byte 13
.byte 12
.byte 11
.byte 10
.byte 9
.byte 8
.byte 7
.byte 6
.byte 5
.byte 4
.byte 3
.byte 2
.byte 1
#endif /* } USE_ROLL_ASM */