Date: Mon, 1 Feb 2021 12:41:03 GMT From: Mateusz Guzik <mjg@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org Subject: git: e5b674594e64 - stable/13 - Reimplement strlen Message-ID: <202102011241.111Cf3hj098420@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch stable/13 has been updated by mjg: URL: https://cgit.FreeBSD.org/src/commit/?id=e5b674594e64b86a34295a475bb6cdb67469f8da commit e5b674594e64b86a34295a475bb6cdb67469f8da Author: Mateusz Guzik <mjg@FreeBSD.org> AuthorDate: 2021-01-29 21:48:11 +0000 Commit: Mateusz Guzik <mjg@FreeBSD.org> CommitDate: 2021-02-01 12:39:18 +0000 Reimplement strlen The previous code neglected to use primitives which can find the end of the string without having to branch on every character. While here augment the somewhat misleading commentary -- strlen as implemented here leaves performance on the table, especially so for userspace. Every arch should get a dedicated variant instead. In the meantime this commit lessens the problem. Tested with glibc test suite. Naive test just calling strlen in a loop on Haswell (ops/s): $(perl -e "print 'A' x 3"): before: 211198039 after: 338626619 $(perl -e "print 'A' x 100"): before: 83151997 after: 98285919 (cherry picked from commit 710e45c4b8539d028877769f1a4ec088c48fb5f1) --- lib/libc/string/strlen.c | 82 +++++++++++++++++------------------------------- sys/libkern/strlen.c | 79 +++++++++++++++------------------------------- 2 files changed, 53 insertions(+), 108 deletions(-) diff --git a/lib/libc/string/strlen.c b/lib/libc/string/strlen.c index a862ffc245ca..0bdc81d7bb9a 100644 --- a/lib/libc/string/strlen.c +++ b/lib/libc/string/strlen.c @@ -35,10 +35,6 @@ __FBSDID("$FreeBSD$"); /* * Portable strlen() for 32-bit and 64-bit systems. * - * Rationale: it is generally much more efficient to do word length - * operations and avoid branches on modern computer systems, as - * compared to byte-length operations with a lot of branches. - * * The expression: * * ((x - 0x01....01) & ~x & 0x80....80) @@ -46,18 +42,13 @@ __FBSDID("$FreeBSD$"); * would evaluate to a non-zero value iff any of the bytes in the * original word is zero. * - * On multi-issue processors, we can divide the above expression into: - * a) (x - 0x01....01) - * b) (~x & 0x80....80) - * c) a & b - * - * Where, a) and b) can be partially computed in parallel. - * * The algorithm above is found on "Hacker's Delight" by * Henry S. Warren, Jr. + * + * Note: this leaves performance on the table and each architecture + * would be best served with a tailor made routine instead. */ -/* Magic numbers for the algorithm */ #if LONG_BIT == 32 static const unsigned long mask01 = 0x01010101; static const unsigned long mask80 = 0x80808080; @@ -70,62 +61,45 @@ static const unsigned long mask80 = 0x8080808080808080; #define LONGPTR_MASK (sizeof(long) - 1) -/* - * Helper macro to return string length if we caught the zero - * byte. - */ -#define testbyte(x) \ - do { \ - if (p[x] == '\0') \ - return (p - str + x); \ - } while (0) +#if BYTE_ORDER == LITTLE_ENDIAN +#define FINDZERO __builtin_ctzl +#else +#define FINDZERO __builtin_clzl +#endif size_t strlen(const char *str) { - const char *p; const unsigned long *lp; + unsigned long mask; long va, vb; + long val; - /* - * Before trying the hard (unaligned byte-by-byte access) way - * to figure out whether there is a nul character, try to see - * if there is a nul character is within this accessible word - * first. - * - * p and (p & ~LONGPTR_MASK) must be equally accessible since - * they always fall in the same memory page, as long as page - * boundaries is integral multiple of word size. - */ - lp = (const unsigned long *)((uintptr_t)str & ~LONGPTR_MASK); - va = (*lp - mask01); - vb = ((~*lp) & mask80); - lp++; - if (va & vb) - /* Check if we have \0 in the first part */ - for (p = str; p < (const char *)lp; p++) - if (*p == '\0') - return (p - str); + lp = (unsigned long *) (uintptr_t) str; + if ((uintptr_t)lp & LONGPTR_MASK) { + lp = (__typeof(lp)) ((uintptr_t)lp & ~LONGPTR_MASK); +#if BYTE_ORDER == LITTLE_ENDIAN + mask = ~(~0UL << (((uintptr_t)str & LONGPTR_MASK) << 3)); +#else + mask = ~(~0UL >> (((uintptr_t)str & LONGPTR_MASK) << 3)); +#endif + val = *lp | mask; + va = (val - mask01); + vb = ((~val) & mask80); + if (va & vb) { + return ((const char *)lp - str + (FINDZERO(va & vb) >> 3)); + } + lp++; + } - /* Scan the rest of the string using word sized operation */ for (; ; lp++) { va = (*lp - mask01); vb = ((~*lp) & mask80); if (va & vb) { - p = (const char *)(lp); - testbyte(0); - testbyte(1); - testbyte(2); - testbyte(3); -#if (LONG_BIT >= 64) - testbyte(4); - testbyte(5); - testbyte(6); - testbyte(7); -#endif + return ((const char *)lp - str + (FINDZERO(va & vb) >> 3)); } } - /* NOTREACHED */ + __builtin_unreachable(); return (0); } diff --git a/sys/libkern/strlen.c b/sys/libkern/strlen.c index a8c7964f69a3..8fa5f3927ea9 100644 --- a/sys/libkern/strlen.c +++ b/sys/libkern/strlen.c @@ -34,10 +34,6 @@ __FBSDID("$FreeBSD$"); /* * Portable strlen() for 32-bit and 64-bit systems. * - * Rationale: it is generally much more efficient to do word length - * operations and avoid branches on modern computer systems, as - * compared to byte-length operations with a lot of branches. - * * The expression: * * ((x - 0x01....01) & ~x & 0x80....80) @@ -45,18 +41,10 @@ __FBSDID("$FreeBSD$"); * would evaluate to a non-zero value iff any of the bytes in the * original word is zero. * - * On multi-issue processors, we can divide the above expression into: - * a) (x - 0x01....01) - * b) (~x & 0x80....80) - * c) a & b - * - * Where, a) and b) can be partially computed in parallel. - * * The algorithm above is found on "Hacker's Delight" by * Henry S. Warren, Jr. */ -/* Magic numbers for the algorithm */ #if LONG_BIT == 32 static const unsigned long mask01 = 0x01010101; static const unsigned long mask80 = 0x80808080; @@ -69,62 +57,45 @@ static const unsigned long mask80 = 0x8080808080808080; #define LONGPTR_MASK (sizeof(long) - 1) -/* - * Helper macro to return string length if we caught the zero - * byte. - */ -#define testbyte(x) \ - do { \ - if (p[x] == '\0') \ - return (p - str + x); \ - } while (0) +#if BYTE_ORDER == LITTLE_ENDIAN +#define FINDZERO __builtin_ctzl +#else +#define FINDZERO __builtin_clzl +#endif size_t (strlen)(const char *str) { - const char *p; const unsigned long *lp; + unsigned long mask; long va, vb; + long val; - /* - * Before trying the hard (unaligned byte-by-byte access) way - * to figure out whether there is a nul character, try to see - * if there is a nul character is within this accessible word - * first. - * - * p and (p & ~LONGPTR_MASK) must be equally accessible since - * they always fall in the same memory page, as long as page - * boundaries is integral multiple of word size. - */ - lp = (const unsigned long *)((uintptr_t)str & ~LONGPTR_MASK); - va = (*lp - mask01); - vb = ((~*lp) & mask80); - lp++; - if (va & vb) - /* Check if we have \0 in the first part */ - for (p = str; p < (const char *)lp; p++) - if (*p == '\0') - return (p - str); + lp = (unsigned long *) (uintptr_t) str; + if ((uintptr_t)lp & LONGPTR_MASK) { + lp = (__typeof(lp)) ((uintptr_t)lp & ~LONGPTR_MASK); +#if BYTE_ORDER == LITTLE_ENDIAN + mask = ~(~0UL << (((uintptr_t)str & LONGPTR_MASK) << 3)); +#else + mask = ~(~0UL >> (((uintptr_t)str & LONGPTR_MASK) << 3)); +#endif + val = *lp | mask; + va = (val - mask01); + vb = ((~val) & mask80); + if (va & vb) { + return ((const char *)lp - str + (FINDZERO(va & vb) >> 3)); + } + lp++; + } - /* Scan the rest of the string using word sized operation */ for (; ; lp++) { va = (*lp - mask01); vb = ((~*lp) & mask80); if (va & vb) { - p = (const char *)(lp); - testbyte(0); - testbyte(1); - testbyte(2); - testbyte(3); -#if (LONG_BIT >= 64) - testbyte(4); - testbyte(5); - testbyte(6); - testbyte(7); -#endif + return ((const char *)lp - str + (FINDZERO(va & vb) >> 3)); } } - /* NOTREACHED */ + __builtin_unreachable(); return (0); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202102011241.111Cf3hj098420>