Date: Thu, 9 Nov 2023 18:21:47 GMT From: Alexander Motin <mav@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 7c566d6cfc7b - main - uma: Micro-optimize memory trashing Message-ID: <202311091821.3A9ILlAJ044944@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by mav: URL: https://cgit.FreeBSD.org/src/commit/?id=7c566d6cfc7bfb913bad89d87386fa21dce8c2e6 commit 7c566d6cfc7bfb913bad89d87386fa21dce8c2e6 Author: Alexander Motin <mav@FreeBSD.org> AuthorDate: 2023-11-09 18:07:46 +0000 Commit: Alexander Motin <mav@FreeBSD.org> CommitDate: 2023-11-09 18:07:46 +0000 uma: Micro-optimize memory trashing Use u_long for memory accesses instead of uint32_t. On my tests on amd64 this by ~30% reduces time spent in those functions thanks to bigger 64bit accesses. i386 still uses 32bit accesses. MFC after: 1 month --- sys/vm/uma_dbg.c | 62 ++++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/sys/vm/uma_dbg.c b/sys/vm/uma_dbg.c index 36567f3b3968..76dd2bfde2fe 100644 --- a/sys/vm/uma_dbg.c +++ b/sys/vm/uma_dbg.c @@ -53,7 +53,7 @@ #include <vm/uma_dbg.h> #include <vm/memguard.h> -static const uint32_t uma_junk = 0xdeadc0de; +static const u_long uma_junk = (u_long)0xdeadc0dedeadc0de; /* * Checks an item to make sure it hasn't been overwritten since it was freed, @@ -64,27 +64,20 @@ static const uint32_t uma_junk = 0xdeadc0de; int trash_ctor(void *mem, int size, void *arg, int flags) { - int cnt; - uint32_t *p; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) return (0); #endif - cnt = size / sizeof(uma_junk); - - for (p = mem; cnt > 0; cnt--, p++) - if (*p != uma_junk) { -#ifdef INVARIANTS - panic("Memory modified after free %p(%d) val=%x @ %p\n", - mem, size, *p, p); -#else - printf("Memory modified after free %p(%d) val=%x @ %p\n", - mem, size, *p, p); -#endif - return (0); - } + e = p + size / sizeof(*p); + for (; p < e; p++) { + if (__predict_true(*p == uma_junk)) + continue; + panic("Memory modified after free %p(%d) val=%lx @ %p\n", + mem, size, *p, p); + } return (0); } @@ -97,17 +90,15 @@ trash_ctor(void *mem, int size, void *arg, int flags) void trash_dtor(void *mem, int size, void *arg) { - int cnt; - uint32_t *p; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) return; #endif - cnt = size / sizeof(uma_junk); - - for (p = mem; cnt > 0; cnt--, p++) + e = p + size / sizeof(*p); + for (; p < e; p++) *p = uma_junk; } @@ -140,8 +131,7 @@ int mtrash_ctor(void *mem, int size, void *arg, int flags) { struct malloc_type **ksp; - uint32_t *p = mem; - int cnt; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) @@ -151,15 +141,16 @@ mtrash_ctor(void *mem, int size, void *arg, int flags) size -= sizeof(struct malloc_type *); ksp = (struct malloc_type **)mem; ksp += size / sizeof(struct malloc_type *); - cnt = size / sizeof(uma_junk); - - for (p = mem; cnt > 0; cnt--, p++) - if (*p != uma_junk) { - printf("Memory modified after free %p(%d) val=%x @ %p\n", - mem, size, *p, p); - panic("Most recently used by %s\n", (*ksp == NULL)? - "none" : (*ksp)->ks_shortdesc); - } + + e = p + size / sizeof(*p); + for (; p < e; p++) { + if (__predict_true(*p == uma_junk)) + continue; + printf("Memory modified after free %p(%d) val=%lx @ %p\n", + mem, size, *p, p); + panic("Most recently used by %s\n", (*ksp == NULL)? + "none" : (*ksp)->ks_shortdesc); + } return (0); } @@ -172,8 +163,7 @@ mtrash_ctor(void *mem, int size, void *arg, int flags) void mtrash_dtor(void *mem, int size, void *arg) { - int cnt; - uint32_t *p; + u_long *p = mem, *e; #ifdef DEBUG_MEMGUARD if (is_memguard_addr(mem)) @@ -181,9 +171,9 @@ mtrash_dtor(void *mem, int size, void *arg) #endif size -= sizeof(struct malloc_type *); - cnt = size / sizeof(uma_junk); - for (p = mem; cnt > 0; cnt--, p++) + e = p + size / sizeof(*p); + for (; p < e; p++) *p = uma_junk; }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202311091821.3A9ILlAJ044944>