Kernel/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-12-29 17:25:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

x86/csum: clean up `csum_partial' further

Browse Source 
Commit 688eb8191b ("x86/csum: Improve performance of `csum_partial`")
ended up improving the code generation for the IP csum calculations, and
in particular special-casing the 40-byte case that is a hot case for
IPv6 headers.

It then had _another_ special case for the 64-byte unrolled loop, which
did two chains of 32-byte blocks, which allows modern CPU's to improve
performance by doing the chains in parallel thanks to renaming the carry
flag.

This just unifies the special cases and combines them into just one
single helper the 40-byte csum case, and replaces the 64-byte case by a
80-byte case that just does that single helper twice.  It avoids having
all these different versions of inline assembly, and actually improved
performance further in my tests.

There was never anything magical about the 64-byte unrolled case, even
though it happens to be a common size (and typically is the cacheline
size).

Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Linus Torvalds 2023-06-27 13:55:32 -07:00
parent 5d4acb6285
commit a476aae3f1
1 changed files with 38 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
arch/x86/lib/csum-partial_64.c
View File

@ -16,6 +16,20 @@ static inline __wsum csum_finalize_sum(u64 temp64)
return (__force __wsum)((temp64 + ror64(temp64, 32)) >> 32);
}
static inline unsigned long update_csum_40b(unsigned long sum, const unsigned long m[5])
{
asm("addq %1,%0\n\t"
"adcq %2,%0\n\t"
"adcq %3,%0\n\t"
"adcq %4,%0\n\t"
"adcq %5,%0\n\t"
"adcq $0,%0"
:"+r" (sum)
:"m" (m[0]), "m" (m[1]), "m" (m[2]),
"m" (m[3]), "m" (m[4]));
return sum;
}
/*
* Do a checksum on an arbitrary memory area.
* Returns a 32bit checksum.
@ -31,52 +45,31 @@ __wsum csum_partial(const void *buff, int len, __wsum sum)
{
u64 temp64 = (__force u64)sum;
/*
* len == 40 is the hot case due to IPv6 headers, but annotating it likely()
* has noticeable negative affect on codegen for all other cases with
* minimal performance benefit here.
*/
if (len == 40) {
asm("addq 0*8(%[src]),%[res]\n\t"
"adcq 1*8(%[src]),%[res]\n\t"
"adcq 2*8(%[src]),%[res]\n\t"
"adcq 3*8(%[src]),%[res]\n\t"
"adcq 4*8(%[src]),%[res]\n\t"
"adcq $0,%[res]"
: [res] "+r"(temp64)
: [src] "r"(buff), "m"(*(const char(*)[40])buff));
return csum_finalize_sum(temp64);
}
if (unlikely(len >= 64)) {
/*
* Extra accumulators for better ILP in the loop.
*/
u64 tmp_accum, tmp_carries;
/* Do two 40-byte chunks in parallel to get better ILP */
if (likely(len >= 80)) {
u64 temp64_2 = 0;
do {
temp64 = update_csum_40b(temp64, buff);
temp64_2 = update_csum_40b(temp64_2, buff + 40);
buff += 80;
len -= 80;
} while (len >= 80);
asm("xorl %k[tmp_accum],%k[tmp_accum]\n\t"
"xorl %k[tmp_carries],%k[tmp_carries]\n\t"
"subl $64, %[len]\n\t"
"1:\n\t"
"addq 0*8(%[src]),%[res]\n\t"
"adcq 1*8(%[src]),%[res]\n\t"
"adcq 2*8(%[src]),%[res]\n\t"
"adcq 3*8(%[src]),%[res]\n\t"
"adcl $0,%k[tmp_carries]\n\t"
"addq 4*8(%[src]),%[tmp_accum]\n\t"
"adcq 5*8(%[src]),%[tmp_accum]\n\t"
"adcq 6*8(%[src]),%[tmp_accum]\n\t"
"adcq 7*8(%[src]),%[tmp_accum]\n\t"
"adcl $0,%k[tmp_carries]\n\t"
"addq $64, %[src]\n\t"
"subl $64, %[len]\n\t"
"jge 1b\n\t"
"addq %[tmp_accum],%[res]\n\t"
"adcq %[tmp_carries],%[res]\n\t"
"adcq $0,%[res]"
: [tmp_accum] "=&r"(tmp_accum),
[tmp_carries] "=&r"(tmp_carries), [res] "+r"(temp64),
[len] "+r"(len), [src] "+r"(buff)
: "m"(*(const char *)buff));
asm("addq %1,%0\n\t"
"adcq $0,%0"
:"+r" (temp64): "r" (temp64_2));
}
/*
* len == 40 is the hot case due to IPv6 headers, so return
* early for that exact case without checking the tail bytes.
*/
if (len >= 40) {
temp64 = update_csum_40b(temp64, buff);
len -= 40;
if (!len)
return csum_finalize_sum(temp64);
buff += 40;
}
if (len & 32) {