Date: Fri, 9 Feb 2018 04:45:39 +0000 (UTC) From: Gleb Smirnoff <glebius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r329058 - in head/sys: kern vm Message-ID: <201802090445.w194jduj089409@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: glebius Date: Fri Feb 9 04:45:39 2018 New Revision: 329058 URL: https://svnweb.freebsd.org/changeset/base/329058 Log: Fix boot_pages exhaustion on machines with many domains and cores, where size of UMA zone allocation is greater than page size. In this case zone of zones can not use UMA_MD_SMALL_ALLOC, and we need to postpone switch off of this zone from startup_alloc() until full launch of VM. o Always supply number of VM zones to uma_startup_count(). On machines with UMA_MD_SMALL_ALLOC ignore it completely, unless zsize goes over a page. In the latter case account VM zones for number of allocations from the zone of zones. o Rewrite startup_alloc() so that it will immediately switch off from itself any zone that is already capable of running real alloc. In worst case scenario we may leak a single page here. See comment in uma_startup_count(). o Hardcode call to uma_startup2() into vm_mem_init(). Otherwise some extra SYSINITs, e.g. vm_page_init() may sneak in before. o While here, remove uma_boot_pages_mtx. With recent changes to boot pages calculation, we are guaranteed to use all of the boot_pages in the early single threaded stage. Reported & tested by: mav Modified: head/sys/kern/kern_malloc.c head/sys/vm/uma_core.c head/sys/vm/vm_init.c head/sys/vm/vm_page.c Modified: head/sys/kern/kern_malloc.c ============================================================================== --- head/sys/kern/kern_malloc.c Fri Feb 9 03:07:12 2018 (r329057) +++ head/sys/kern/kern_malloc.c Fri Feb 9 04:45:39 2018 (r329058) @@ -96,8 +96,6 @@ __FBSDID("$FreeBSD$"); dtrace_malloc_probe_func_t dtrace_malloc_probe; #endif -extern void uma_startup2(void); - #if defined(INVARIANTS) || defined(MALLOC_MAKE_FAILURES) || \ defined(DEBUG_MEMGUARD) || defined(DEBUG_REDZONE) #define MALLOC_DEBUG 1 @@ -928,8 +926,6 @@ mallocinit(void *dummy) mtx_init(&malloc_mtx, "malloc", NULL, MTX_DEF); kmeminit(); - - uma_startup2(); if (kmem_zmax < PAGE_SIZE || kmem_zmax > KMEM_ZMAX) kmem_zmax = KMEM_ZMAX; Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Fri Feb 9 03:07:12 2018 (r329057) +++ head/sys/vm/uma_core.c Fri Feb 9 04:45:39 2018 (r329058) @@ -134,13 +134,10 @@ static struct rwlock_padalign __exclusive_cache_line u /* * Pointer and counter to pool of pages, that is preallocated at - * startup to bootstrap UMA. Early zones continue to use the pool - * until it is depleted, so allocations may happen after boot, thus - * we need a mutex to protect it. + * startup to bootstrap UMA. */ static char *bootmem; static int boot_pages; -static struct mtx uma_boot_pages_mtx; static struct sx uma_drain_lock; @@ -1081,37 +1078,46 @@ startup_alloc(uma_zone_t zone, vm_size_t bytes, int do int pages; keg = zone_first_keg(zone); - pages = howmany(bytes, PAGE_SIZE); - KASSERT(pages > 0, ("startup_alloc can't reserve 0 pages\n")); /* - * Check our small startup cache to see if it has pages remaining. + * If we are in BOOT_BUCKETS or higher, than switch to real + * allocator. Zones with page sized slabs switch at BOOT_PAGEALLOC. */ - mtx_lock(&uma_boot_pages_mtx); - if (pages <= boot_pages) { -#ifdef DIAGNOSTIC - printf("%s from \"%s\", %d boot pages left\n", __func__, - zone->uz_name, boot_pages); + switch (booted) { + case BOOT_COLD: + case BOOT_STRAPPED: + break; + case BOOT_PAGEALLOC: + if (keg->uk_ppera > 1) + break; + case BOOT_BUCKETS: + case BOOT_RUNNING: +#ifdef UMA_MD_SMALL_ALLOC + keg->uk_allocf = (keg->uk_ppera > 1) ? + page_alloc : uma_small_alloc; +#else + keg->uk_allocf = page_alloc; #endif - mem = bootmem; - boot_pages -= pages; - bootmem += pages * PAGE_SIZE; - mtx_unlock(&uma_boot_pages_mtx); - *pflag = UMA_SLAB_BOOT; - return (mem); + return keg->uk_allocf(zone, bytes, domain, pflag, wait); } - mtx_unlock(&uma_boot_pages_mtx); - if (booted < BOOT_PAGEALLOC) - panic("UMA zone \"%s\": Increase vm.boot_pages", zone->uz_name); + /* - * Now that we've booted reset these users to their real allocator. + * Check our small startup cache to see if it has pages remaining. */ -#ifdef UMA_MD_SMALL_ALLOC - keg->uk_allocf = (keg->uk_ppera > 1) ? page_alloc : uma_small_alloc; -#else - keg->uk_allocf = page_alloc; + pages = howmany(bytes, PAGE_SIZE); + KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); + if (pages > boot_pages) + panic("UMA zone \"%s\": Increase vm.boot_pages", zone->uz_name); +#ifdef DIAGNOSTIC + printf("%s from \"%s\", %d boot pages left\n", __func__, zone->uz_name, + boot_pages); #endif - return keg->uk_allocf(zone, bytes, domain, pflag, wait); + mem = bootmem; + boot_pages -= pages; + bootmem += pages * PAGE_SIZE; + *pflag = UMA_SLAB_BOOT; + + return (mem); } /* @@ -1789,9 +1795,9 @@ zone_foreach(void (*zfunc)(uma_zone_t)) #define UMA_BOOT_ALIGN 32 static int zsize, ksize; int -uma_startup_count(int zones) +uma_startup_count(int vm_zones) { - int pages; + int zones, pages; ksize = sizeof(struct uma_keg) + (sizeof(struct uma_domain) * vm_ndomains); @@ -1806,12 +1812,17 @@ uma_startup_count(int zones) pages = howmany(roundup(zsize, CACHE_LINE_SIZE) * 2 + roundup(ksize, CACHE_LINE_SIZE), PAGE_SIZE); - zones += UMA_BOOT_ZONES; +#ifdef UMA_MD_SMALL_ALLOC + zones = UMA_BOOT_ZONES; +#else + zones = UMA_BOOT_ZONES + vm_zones; + vm_zones = 0; +#endif /* Memory for the rest of startup zones, UMA and VM, ... */ if (zsize > UMA_SLAB_SIZE) - pages += zones * howmany(roundup2(zsize, UMA_BOOT_ALIGN), - UMA_SLAB_SIZE); + pages += (zones + vm_zones) * + howmany(roundup2(zsize, UMA_BOOT_ALIGN), UMA_SLAB_SIZE); else pages += howmany(zones, UMA_SLAB_SPACE / roundup2(zsize, UMA_BOOT_ALIGN)); @@ -1872,7 +1883,6 @@ uma_startup(void *mem, int npages) args.flags = UMA_ZFLAG_INTERNAL; zone_ctor(kegs, zsize, &args, M_WAITOK); - mtx_init(&uma_boot_pages_mtx, "UMA boot pages", NULL, MTX_DEF); bootmem = mem; boot_pages = npages; @@ -1917,6 +1927,9 @@ void uma_startup2(void) { +#ifdef DIAGNOSTIC + printf("Entering %s with %d boot pages left\n", __func__, boot_pages); +#endif booted = BOOT_BUCKETS; sx_init(&uma_drain_lock, "umadrain"); bucket_enable(); Modified: head/sys/vm/vm_init.c ============================================================================== --- head/sys/vm/vm_init.c Fri Feb 9 03:07:12 2018 (r329057) +++ head/sys/vm/vm_init.c Fri Feb 9 04:45:39 2018 (r329058) @@ -95,6 +95,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_extern.h> extern void uma_startup1(void); +extern void uma_startup2(void); extern void vm_radix_reserve_kva(void); #if VM_NRESERVLEVEL > 0 @@ -183,9 +184,9 @@ vm_mem_init(dummy) #ifndef UMA_MD_SMALL_ALLOC /* Set up radix zone to use noobj_alloc. */ vm_radix_reserve_kva(); - /* Announce page availability to UMA. */ - uma_startup1(); #endif + /* Announce full page availability to UMA. */ + uma_startup2(); kmem_init_zero_region(); pmap_init(); vm_pager_init(); Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Fri Feb 9 03:07:12 2018 (r329057) +++ head/sys/vm/vm_page.c Fri Feb 9 04:45:39 2018 (r329058) @@ -506,16 +506,13 @@ vm_page_startup(vm_offset_t vaddr) * Allocate memory for use when boot strapping the kernel memory * allocator. Tell UMA how many zones we are going to create * before going fully functional. UMA will add its zones. - */ -#ifdef UMA_MD_SMALL_ALLOC - boot_pages = uma_startup_count(0); -#else - /* + * * VM startup zones: vmem, vmem_btag, VM OBJECT, RADIX NODE, MAP, * KMAP ENTRY, MAP ENTRY, VMSPACE. */ boot_pages = uma_startup_count(8); +#ifndef UMA_MD_SMALL_ALLOC /* vmem_startup() calls uma_prealloc(). */ boot_pages += vmem_startup_count(); /* vm_map_startup() calls uma_prealloc(). */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201802090445.w194jduj089409>