Date: Wed, 20 Dec 2017 04:02:58 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r327014 - in user/jeff/numa/sys: kern sys vm Message-ID: <201712200402.vBK42wst000324@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Wed Dec 20 04:02:58 2017 New Revision: 327014 URL: https://svnweb.freebsd.org/changeset/base/327014 Log: Add domain specific malloc functions. Modified: user/jeff/numa/sys/kern/kern_malloc.c user/jeff/numa/sys/sys/malloc.h user/jeff/numa/sys/vm/uma_core.c user/jeff/numa/sys/vm/uma_int.h Modified: user/jeff/numa/sys/kern/kern_malloc.c ============================================================================== --- user/jeff/numa/sys/kern/kern_malloc.c Wed Dec 20 01:03:34 2017 (r327013) +++ user/jeff/numa/sys/kern/kern_malloc.c Wed Dec 20 04:02:58 2017 (r327014) @@ -95,6 +95,11 @@ __FBSDID("$FreeBSD$"); dtrace_malloc_probe_func_t dtrace_malloc_probe; #endif +#if defined(INVARIANTS) || defined(MALLOC_MAKE_FAILURES) || \ + defined(DEBUG_MEMGUARD) || defined(DEBUG_REDZONE) +#define MALLOC_DEBUG 1 +#endif + /* * When realloc() is called, if the new size is sufficiently smaller than * the old size, realloc() will allocate a new, smaller block to avoid @@ -416,6 +421,20 @@ contigmalloc(unsigned long size, struct malloc_type *t return (ret); } +void * +contigmalloc_domain(unsigned long size, struct malloc_type *type, + int domain, int flags, vm_paddr_t low, vm_paddr_t high, + unsigned long alignment, vm_paddr_t boundary) +{ + void *ret; + + ret = (void *)kmem_alloc_contig_domain(domain, size, flags, low, high, + alignment, boundary, VM_MEMATTR_DEFAULT); + if (ret != NULL) + malloc_type_allocated(type, round_page(size)); + return (ret); +} + /* * contigfree: * @@ -431,26 +450,14 @@ contigfree(void *addr, unsigned long size, struct mall malloc_type_freed(type, round_page(size)); } -/* - * malloc: - * - * Allocate a block of memory. - * - * If M_NOWAIT is set, this routine will not block and return NULL if - * the allocation fails. - */ -void * -malloc(unsigned long size, struct malloc_type *mtp, int flags) +#ifdef MALLOC_DEBUG +static int +malloc_dbg(caddr_t *vap, unsigned long *sizep, struct malloc_type *mtp, + int flags) { +#ifdef INVARIANTS int indx; - struct malloc_type_internal *mtip; - caddr_t va; - uma_zone_t zone; -#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) - unsigned long osize = size; -#endif -#ifdef INVARIANTS KASSERT(mtp->ks_magic == M_MAGIC, ("malloc: bad malloc type magic")); /* * Check that exactly one of M_WAITOK or M_NOWAIT is specified. @@ -473,7 +480,8 @@ malloc(unsigned long size, struct malloc_type *mtp, in if ((malloc_nowait_count % malloc_failure_rate) == 0) { atomic_add_int(&malloc_failure_count, 1); t_malloc_fail = time_uptime; - return (NULL); + *vap = NULL; + return (EJUSTRETURN); } } #endif @@ -486,16 +494,46 @@ malloc(unsigned long size, struct malloc_type *mtp, in #ifdef DEBUG_MEMGUARD if (memguard_cmp_mtp(mtp, size)) { va = memguard_alloc(size, flags); - if (va != NULL) - return (va); + if (va != NULL) { + *vap = va; + return (EJUSTRETURN); + } /* This is unfortunate but should not be fatal. */ } #endif #ifdef DEBUG_REDZONE - size = redzone_size_ntor(size); + *sizep = redzone_size_ntor(*sizep); #endif + return (0); +} +#endif + +/* + * malloc: + * + * Allocate a block of memory. + * + * If M_NOWAIT is set, this routine will not block and return NULL if + * the allocation fails. + */ +void * +malloc(unsigned long size, struct malloc_type *mtp, int flags) +{ + int indx; + struct malloc_type_internal *mtip; + caddr_t va; + uma_zone_t zone; +#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + +#ifdef MALLOC_DEBUG + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + if (size <= kmem_zmax) { mtip = mtp->ks_handle; if (size & KMEM_ZMASK) @@ -522,11 +560,55 @@ malloc(unsigned long size, struct malloc_type *mtp, in KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); else if (va == NULL) t_malloc_fail = time_uptime; -#ifdef DIAGNOSTIC - if (va != NULL && !(flags & M_ZERO)) { - memset(va, 0x70, osize); - } +#ifdef DEBUG_REDZONE + if (va != NULL) + va = redzone_setup(va, osize); #endif + return ((void *) va); +} + +void * +malloc_domain(unsigned long size, struct malloc_type *mtp, int domain, + int flags) +{ + int indx; + struct malloc_type_internal *mtip; + caddr_t va; + uma_zone_t zone; +#if defined(DIAGNOSTIC) || defined(DEBUG_REDZONE) + unsigned long osize = size; +#endif + +#ifdef MALLOC_DEBUG + if (malloc_dbg(&va, &size, mtp, flags) != 0) + return (va); +#endif + if (size <= kmem_zmax) { + mtip = mtp->ks_handle; + if (size & KMEM_ZMASK) + size = (size & ~KMEM_ZMASK) + KMEM_ZBASE; + indx = kmemsize[size >> KMEM_ZSHIFT]; + KASSERT(mtip->mti_zone < numzones, + ("mti_zone %u out of range %d", + mtip->mti_zone, numzones)); + zone = kmemzones[indx].kz_zone[mtip->mti_zone]; +#ifdef MALLOC_PROFILE + krequests[size >> KMEM_ZSHIFT]++; +#endif + va = uma_zalloc_domain(zone, NULL, domain, flags); + if (va != NULL) + size = zone->uz_size; + malloc_type_zone_allocated(mtp, va == NULL ? 0 : size, indx); + } else { + size = roundup(size, PAGE_SIZE); + zone = NULL; + va = uma_large_malloc_domain(size, domain, flags); + malloc_type_allocated(mtp, va == NULL ? 0 : size); + } + if (flags & M_WAITOK) + KASSERT(va != NULL, ("malloc(M_WAITOK) returned NULL")); + else if (va == NULL) + t_malloc_fail = time_uptime; #ifdef DEBUG_REDZONE if (va != NULL) va = redzone_setup(va, osize); @@ -534,66 +616,124 @@ malloc(unsigned long size, struct malloc_type *mtp, in return ((void *) va); } -/* - * free: - * - * Free a block of memory allocated by malloc. - * - * This routine may not block. - */ -void -free(void *addr, struct malloc_type *mtp) +#ifdef INVARIANTS +static void +free_save_type(void *addr, struct malloc_type *mtp, u_long size) { - uma_slab_t slab; - u_long size; + struct malloc_type **mtpp = addr; + /* + * Cache a pointer to the malloc_type that most recently freed + * this memory here. This way we know who is most likely to + * have stepped on it later. + * + * This code assumes that size is a multiple of 8 bytes for + * 64 bit machines + */ + mtpp = (struct malloc_type **) ((unsigned long)mtpp & ~UMA_ALIGN_PTR); + mtpp += (size - sizeof(struct malloc_type *)) / + sizeof(struct malloc_type *); + *mtpp = mtp; +} +#endif + +#ifdef MALLOC_DEBUG +static int +free_dbg(void **addrp, struct malloc_type *mtp) +{ + void *addr; + + addr = *addrp; KASSERT(mtp->ks_magic == M_MAGIC, ("free: bad malloc type magic")); KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), ("free: called with spinlock or critical section held")); /* free(NULL, ...) does nothing */ if (addr == NULL) - return; + return (EJUSTRETURN); #ifdef DEBUG_MEMGUARD if (is_memguard_addr(addr)) { memguard_free(addr); - return; + return (EJUSTRETURN); } #endif #ifdef DEBUG_REDZONE redzone_check(addr); - addr = redzone_addr_ntor(addr); + *addrp = redzone_addr_ntor(addr); #endif - slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); + return (0); +} +#endif +/* + * free: + * + * Free a block of memory allocated by malloc. + * + * This routine may not block. + */ +void +free(void *addr, struct malloc_type *mtp) +{ + uma_slab_t slab; + u_long size; + +#ifdef MALLOC_DEBUG + if (free_dbg(&addr, mtp) != 0) + return; +#endif + /* free(NULL, ...) does nothing */ + if (addr == NULL) + return; + + slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); if (slab == NULL) panic("free: address %p(%p) has not been allocated.\n", addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); if (!(slab->us_flags & UMA_SLAB_MALLOC)) { + size = slab->us_keg->uk_size; #ifdef INVARIANTS - struct malloc_type **mtpp = addr; + free_save_type(addr, mtp, size); #endif + uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab); + } else { + size = slab->us_size; + uma_large_free(slab); + } + malloc_type_freed(mtp, size); +} + +void +free_domain(void *addr, struct malloc_type *mtp) +{ + uma_slab_t slab; + u_long size; + +#ifdef MALLOC_DEBUG + if (free_dbg(&addr, mtp) != 0) + return; +#endif + + /* free(NULL, ...) does nothing */ + if (addr == NULL) + return; + + slab = vtoslab((vm_offset_t)addr & (~UMA_SLAB_MASK)); + if (slab == NULL) + panic("free_domain: address %p(%p) has not been allocated.\n", + addr, (void *)((u_long)addr & (~UMA_SLAB_MASK))); + + if (!(slab->us_flags & UMA_SLAB_MALLOC)) { size = slab->us_keg->uk_size; #ifdef INVARIANTS - /* - * Cache a pointer to the malloc_type that most recently freed - * this memory here. This way we know who is most likely to - * have stepped on it later. - * - * This code assumes that size is a multiple of 8 bytes for - * 64 bit machines - */ - mtpp = (struct malloc_type **) - ((unsigned long)mtpp & ~UMA_ALIGN_PTR); - mtpp += (size - sizeof(struct malloc_type *)) / - sizeof(struct malloc_type *); - *mtpp = mtp; + free_save_type(addr, mtp, size); #endif - uma_zfree_arg(LIST_FIRST(&slab->us_keg->uk_zones), addr, slab); + uma_zfree_domain(LIST_FIRST(&slab->us_keg->uk_zones), + addr, slab); } else { size = slab->us_size; uma_large_free(slab); Modified: user/jeff/numa/sys/sys/malloc.h ============================================================================== --- user/jeff/numa/sys/sys/malloc.h Wed Dec 20 01:03:34 2017 (r327013) +++ user/jeff/numa/sys/sys/malloc.h Wed Dec 20 04:02:58 2017 (r327014) @@ -174,8 +174,16 @@ void *contigmalloc(unsigned long size, struct malloc_t vm_paddr_t low, vm_paddr_t high, unsigned long alignment, vm_paddr_t boundary) __malloc_like __result_use_check __alloc_size(1) __alloc_align(6); +void *contigmalloc_domain(unsigned long size, struct malloc_type *type, + int domain, int flags, vm_paddr_t low, vm_paddr_t high, + unsigned long alignment, vm_paddr_t boundary) + __malloc_like __result_use_check __alloc_size(1) __alloc_align(6); void free(void *addr, struct malloc_type *type); +void free_domain(void *addr, struct malloc_type *type); void *malloc(unsigned long size, struct malloc_type *type, int flags) + __malloc_like __result_use_check __alloc_size(1); +void *malloc_domain(unsigned long size, struct malloc_type *type, + int domain, int flags) __malloc_like __result_use_check __alloc_size(1); void malloc_init(void *); int malloc_last_fail(void); Modified: user/jeff/numa/sys/vm/uma_core.c ============================================================================== --- user/jeff/numa/sys/vm/uma_core.c Wed Dec 20 01:03:34 2017 (r327013) +++ user/jeff/numa/sys/vm/uma_core.c Wed Dec 20 04:02:58 2017 (r327014) @@ -221,6 +221,8 @@ struct uma_bucket_zone bucket_zones[] = { */ enum zfreeskip { SKIP_NONE = 0, SKIP_DTOR, SKIP_FINI }; +#define UMA_ANYDOMAIN -1 /* Special value for domain search. */ + /* Prototypes.. */ static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); @@ -564,8 +566,8 @@ hash_alloc(struct uma_hash *hash) M_UMAHASH, M_NOWAIT); } else { alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; - hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 0, - M_WAITOK); + hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, + UMA_ANYDOMAIN, M_WAITOK); hash->uh_hashsize = UMA_HASH_SIZE_INIT; } if (hash->uh_slab_hash) { @@ -1878,7 +1880,7 @@ uma_kcreate(uma_zone_t zone, size_t size, uma_init umi args.align = (align == UMA_ALIGN_CACHE) ? uma_align_cache : align; args.flags = flags; args.zone = zone; - return (zone_alloc_item(kegs, &args, 0, M_WAITOK)); + return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK)); } /* See uma.h */ @@ -1935,7 +1937,7 @@ uma_zcreate(const char *name, size_t size, uma_ctor ct sx_slock(&uma_drain_lock); locked = true; } - res = zone_alloc_item(zones, &args, 0, M_WAITOK); + res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); if (locked) sx_sunlock(&uma_drain_lock); return (res); @@ -1970,7 +1972,7 @@ uma_zsecond_create(char *name, uma_ctor ctor, uma_dtor locked = true; } /* XXX Attaches only one keg of potentially many. */ - res = zone_alloc_item(zones, &args, 0, M_WAITOK); + res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); if (locked) sx_sunlock(&uma_drain_lock); return (res); @@ -1997,7 +1999,7 @@ uma_zcache_create(char *name, int size, uma_ctor ctor, args.align = 0; args.flags = flags; - return (zone_alloc_item(zones, &args, 0, M_WAITOK)); + return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK)); } static void @@ -2206,7 +2208,7 @@ zalloc_start: if (zone->uz_flags & UMA_ZONE_NUMA) domain = PCPU_GET(domain); else - domain = 0; + domain = UMA_ANYDOMAIN; /* Short-circuit for zones without buckets and low memory. */ if (zone->uz_count == 0 || bucketdisable) @@ -2248,7 +2250,10 @@ zalloc_start: /* * Check the zone's cache of buckets. */ - zdom = &zone->uz_domain[domain]; + if (domain == UMA_ANYDOMAIN) + zdom = &zone->uz_domain[0]; + else + zdom = &zone->uz_domain[domain]; if ((bucket = LIST_FIRST(&zdom->uzd_buckets)) != NULL) { KASSERT(bucket->ub_cnt != 0, ("uma_zalloc_arg: Returning an empty bucket.")); @@ -2306,6 +2311,28 @@ zalloc_item: return (item); } +void * +uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) +{ + + /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ + random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA); + + /* This is the fast path allocation */ + CTR5(KTR_UMA, + "uma_zalloc_domain thread %x zone %s(%p) domain %d flags %d", + curthread, zone->uz_name, zone, domain, flags); + + if (flags & M_WAITOK) { + WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, + "uma_zalloc_domain: zone \"%s\"", zone->uz_name); + } + KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), + ("uma_zalloc_domain: called with spinlock or critical section held")); + + return (zone_alloc_item(zone, udata, domain, flags)); +} + /* * Find a slab with some space. Prefer slabs that are partially used over those * that are totally full. This helps to reduce fragmentation. @@ -2360,7 +2387,9 @@ keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdo * Round-robin for non first-touch zones when there is more than one * domain. */ - rr = (zone->uz_flags & UMA_ZONE_NUMA) == 0 && vm_ndomains != 1; + if (vm_ndomains == 1) + rdomain = 0; + rr = rdomain == UMA_ANYDOMAIN; if (rr) { keg->uk_cursor = (keg->uk_cursor + 1) % vm_ndomains; domain = start = keg->uk_cursor; @@ -2665,6 +2694,7 @@ zone_alloc_bucket(uma_zone_t zone, void *udata, int do * Arguments * zone The zone to alloc for. * udata The data to be passed to the constructor. + * domain The domain to allocate from or UMA_ANYDOMAIN. * flags M_WAITOK, M_NOWAIT, M_ZERO. * * Returns @@ -2896,6 +2926,25 @@ zfree_item: return; } +void +uma_zfree_domain(uma_zone_t zone, void *item, void *udata) +{ + + /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ + random_harvest_fast_uma(&zone, sizeof(zone), 1, RANDOM_UMA); + + CTR2(KTR_UMA, "uma_zfree_domain thread %x zone %s", curthread, + zone->uz_name); + + KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), + ("uma_zfree_domain: called with spinlock or critical section held")); + + /* uma_zfree(..., NULL) does nothing, to match free(9). */ + if (item == NULL) + return; + zone_free_item(zone, item, udata, SKIP_NONE); +} + static void slab_free_item(uma_keg_t keg, uma_slab_t slab, void *item) { @@ -3342,15 +3391,18 @@ uma_zone_exhausted_nolock(uma_zone_t zone) } void * -uma_large_malloc(vm_size_t size, int wait) +uma_large_malloc_domain(vm_size_t size, int domain, int wait) { vm_offset_t addr; uma_slab_t slab; - slab = zone_alloc_item(slabzone, NULL, 0, wait); + slab = zone_alloc_item(slabzone, NULL, domain, wait); if (slab == NULL) return (NULL); - addr = kmem_malloc(kernel_arena, size, wait); + if (domain == UMA_ANYDOMAIN) + addr = kmem_malloc(kernel_arena, size, wait); + else + addr = kmem_malloc_domain(domain, size, wait); if (addr != 0) { vsetslab(addr, slab); slab->us_data = (void *)addr; @@ -3364,6 +3416,13 @@ uma_large_malloc(vm_size_t size, int wait) } return ((void *)addr); +} + +void * +uma_large_malloc(vm_size_t size, int wait) +{ + + return uma_large_malloc_domain(size, UMA_ANYDOMAIN, wait); } void Modified: user/jeff/numa/sys/vm/uma_int.h ============================================================================== --- user/jeff/numa/sys/vm/uma_int.h Wed Dec 20 01:03:34 2017 (r327013) +++ user/jeff/numa/sys/vm/uma_int.h Wed Dec 20 04:02:58 2017 (r327014) @@ -386,6 +386,7 @@ zone_first_keg(uma_zone_t zone) /* Internal prototypes */ static __inline uma_slab_t hash_sfind(struct uma_hash *hash, uint8_t *data); void *uma_large_malloc(vm_size_t size, int wait); +void *uma_large_malloc_domain(vm_size_t size, int domain, int wait); void uma_large_free(uma_slab_t slab); /* Lock Macros */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201712200402.vBK42wst000324>