Date: Fri, 19 Jan 2018 21:19:57 +0000 (UTC) From: Jeff Roberson <jeff@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r328172 - in user/jeff/numa/sys: kern sys vm Message-ID: <201801192119.w0JLJver013842@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jeff Date: Fri Jan 19 21:19:57 2018 New Revision: 328172 URL: https://svnweb.freebsd.org/changeset/base/328172 Log: Use bitsets for min and severe limit checks so that we can quickly determine if any domains are in shortfall and synchronously sleep on this condition. Provide some new vm_wait_* functions that are more precise for specific scenarios. Modified: user/jeff/numa/sys/kern/subr_vmem.c user/jeff/numa/sys/sys/vmmeter.h user/jeff/numa/sys/vm/vm_glue.c user/jeff/numa/sys/vm/vm_kern.c user/jeff/numa/sys/vm/vm_page.c user/jeff/numa/sys/vm/vm_pageout.h user/jeff/numa/sys/vm/vm_pagequeue.h user/jeff/numa/sys/vm/vm_swapout.c Modified: user/jeff/numa/sys/kern/subr_vmem.c ============================================================================== --- user/jeff/numa/sys/kern/subr_vmem.c Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/kern/subr_vmem.c Fri Jan 19 21:19:57 2018 (r328172) @@ -644,7 +644,7 @@ vmem_bt_alloc(uma_zone_t zone, vm_size_t bytes, int do * possible due to M_USE_RESERVE page allocation. */ if (wait & M_WAITOK) - VM_WAIT; + vm_wait_domain(domain); return (NULL); } mtx_unlock(&vmem_bt_lock); Modified: user/jeff/numa/sys/sys/vmmeter.h ============================================================================== --- user/jeff/numa/sys/sys/vmmeter.h Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/sys/vmmeter.h Fri Jan 19 21:19:57 2018 (r328172) @@ -146,7 +146,11 @@ struct vmmeter { #ifdef _KERNEL +#include <sys/domainset.h> + extern struct vmmeter vm_cnt; +extern domainset_t vm_min_domains; +extern domainset_t vm_severe_domains; #define VM_CNT_ADD(var, x) counter_u64_add(vm_cnt.var, x) #define VM_CNT_INC(var) VM_CNT_ADD(var, 1) @@ -164,8 +168,7 @@ static inline int vm_page_count_severe(void) { - /* XXX */ - return (vm_cnt.v_free_severe > vm_free_count()); + return (!DOMAINSET_EMPTY(&vm_severe_domains)); } /* @@ -181,8 +184,7 @@ static inline int vm_page_count_min(void) { - /* XXX */ - return (vm_cnt.v_free_min > vm_free_count()); + return (!DOMAINSET_EMPTY(&vm_min_domains)); } #endif /* _KERNEL */ Modified: user/jeff/numa/sys/vm/vm_glue.c ============================================================================== --- user/jeff/numa/sys/vm/vm_glue.c Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/vm/vm_glue.c Fri Jan 19 21:19:57 2018 (r328172) @@ -552,7 +552,7 @@ vm_forkproc(struct thread *td, struct proc *p2, struct } while (vm_page_count_severe()) { - VM_WAIT; + vm_wait_severe(); } if ((flags & RFMEM) == 0) { Modified: user/jeff/numa/sys/vm/vm_kern.c ============================================================================== --- user/jeff/numa/sys/vm/vm_kern.c Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/vm/vm_kern.c Fri Jan 19 21:19:57 2018 (r328172) @@ -197,7 +197,7 @@ retry: if (!vm_page_reclaim_contig_domain(domain, pflags, 1, low, high, PAGE_SIZE, 0) && (flags & M_WAITOK) != 0) - VM_WAIT; + vm_wait_domain(domain); VM_OBJECT_WLOCK(object); tries++; goto retry; @@ -281,7 +281,7 @@ retry: if (!vm_page_reclaim_contig_domain(domain, pflags, npages, low, high, alignment, boundary) && (flags & M_WAITOK) != 0) - VM_WAIT; + vm_wait_domain(domain); VM_OBJECT_WLOCK(object); tries++; goto retry; Modified: user/jeff/numa/sys/vm/vm_page.c ============================================================================== --- user/jeff/numa/sys/vm/vm_page.c Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/vm/vm_page.c Fri Jan 19 21:19:57 2018 (r328172) @@ -134,7 +134,11 @@ __FBSDID("$FreeBSD$"); struct vm_domain vm_dom[MAXMEMDOM]; struct mtx_padalign __exclusive_cache_line pa_lock[PA_LOCK_COUNT]; +struct mtx_padalign __exclusive_cache_line vm_domainset_lock; +domainset_t __exclusive_cache_line vm_min_domains; +domainset_t __exclusive_cache_line vm_severe_domains; + /* * bogus page -- for I/O to/from partially complete buffers, * or for paging into sparsely invalid regions. @@ -481,6 +485,7 @@ vm_page_startup(vm_offset_t vaddr) /* * Initialize the page and queue locks. */ + mtx_init(&vm_domainset_lock, "vm domainset lock", NULL, MTX_DEF); for (i = 0; i < PA_LOCK_COUNT; i++) mtx_init(&pa_lock[i], "vm page", NULL, MTX_DEF); for (i = 0; i < vm_ndomains; i++) @@ -2668,14 +2673,85 @@ vm_page_reclaim_contig(int req, u_long npages, vm_padd return (ret); } +/* + * Set the domain in the appropriate page level domainset. + */ +void +vm_domain_set(int domain) +{ + struct vm_domain *vmd; + vmd = VM_DOMAIN(domain); + mtx_lock(&vm_domainset_lock); + if (!vmd->vmd_minset && vm_paging_min(vmd)) { + vmd->vmd_minset = 1; + DOMAINSET_SET(domain, &vm_min_domains); + } + if (!vmd->vmd_severeset && vm_paging_severe(vmd)) { + vmd->vmd_severeset = 1; + DOMAINSET_CLR(domain, &vm_severe_domains); + } + mtx_unlock(&vm_domainset_lock); +} + /* - * vm_wait: (also see VM_WAIT macro) + * Clear the domain from the appropriate page level domainset. + */ +static void +vm_domain_clear(int domain) +{ + struct vm_domain *vmd; + + vmd = VM_DOMAIN(domain); + mtx_lock(&vm_domainset_lock); + if (vmd->vmd_minset && !vm_paging_min(vmd)) { + vmd->vmd_minset = 0; + DOMAINSET_CLR(domain, &vm_min_domains); + if (!vm_page_count_min()) + wakeup(&vm_min_domains); + } + if (vmd->vmd_severeset && !vm_paging_severe(vmd)) { + vmd->vmd_severeset = 0; + DOMAINSET_CLR(domain, &vm_severe_domains); + if (!vm_page_count_severe()) + wakeup(&vm_severe_domains); + } + mtx_unlock(&vm_domainset_lock); +} + +/* + * Wait for free pages to exceed the min threshold globally. + */ +void +vm_wait_min(void) +{ + + mtx_lock(&vm_domainset_lock); + while (vm_page_count_min()) + msleep(&vm_min_domains, &vm_domainset_lock, PVM, "vmwait", 0); + mtx_unlock(&vm_domainset_lock); +} + +/* + * Wait for free pages to exceed the severe threshold globally. + */ +void +vm_wait_severe(void) +{ + + mtx_lock(&vm_domainset_lock); + while (vm_page_count_min()) + msleep(&vm_min_domains, &vm_domainset_lock, PVM, "vmwait", 0); + mtx_unlock(&vm_domainset_lock); +} + +/* + * vm_wait_domain: * * Sleep until free pages are available for allocation. - * - Called in various places before memory allocations. + * - Called in various places after failed memory allocations. */ -static void +void vm_wait_domain(int domain) { struct vm_domain *vmd; @@ -2694,6 +2770,12 @@ vm_wait_domain(int domain) } } +/* + * vm_wait: (also see VM_WAIT macro) + * + * Sleep until free pages are available for allocation. + * - Called in various places after failed memory allocations. + */ void vm_wait(void) { @@ -2942,6 +3024,9 @@ vm_page_free_wakeup(int domain) vmd->vmd_pages_needed = false; wakeup(&vmd->vmd_free_count); } + if ((vmd->vmd_minset && !vm_paging_min(vmd)) || + (vmd->vmd_severeset && !vm_paging_severe(vmd))) + vm_domain_clear(domain); } /* Modified: user/jeff/numa/sys/vm/vm_pageout.h ============================================================================== --- user/jeff/numa/sys/vm/vm_pageout.h Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/vm/vm_pageout.h Fri Jan 19 21:19:57 2018 (r328172) @@ -99,6 +99,9 @@ void pagedaemon_wakeup(int domain); #define VM_WAITPFAULT vm_waitpfault() void vm_wait(void); void vm_waitpfault(void); +void vm_wait_domain(int domain); +void vm_wait_min(void); +void vm_wait_severe(void); #ifdef _KERNEL int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *); Modified: user/jeff/numa/sys/vm/vm_pagequeue.h ============================================================================== --- user/jeff/numa/sys/vm/vm_pagequeue.h Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/vm/vm_pagequeue.h Fri Jan 19 21:19:57 2018 (r328172) @@ -92,6 +92,8 @@ struct vm_domain { int vmd_pageout_deficit; /* Estimated number of pages deficit */ bool vmd_pages_needed; /* Are threads waiting for free pages? */ bool vmd_pageout_wanted; /* pageout daemon wait channel */ + bool vmd_minset; /* Are we in vm_min_domains? */ + bool vmd_severeset; /* Are we in vm_severe_domains? */ int vmd_inactq_scans; enum { VM_LAUNDRY_IDLE = 0, @@ -107,7 +109,7 @@ struct vm_domain { u_int vmd_pageout_wakeup_thresh;/* (c) min pages to wake pagedaemon */ u_int vmd_interrupt_free_min; /* (c) reserved pages for int code */ u_int vmd_free_severe; /* (c) severe page depletion point */ -}; +} __aligned(CACHE_LINE_SIZE); extern struct vm_domain vm_dom[MAXMEMDOM]; @@ -141,14 +143,8 @@ vm_pagequeue_cnt_add(struct vm_pagequeue *pq, int adde #define vm_pagequeue_cnt_inc(pq) vm_pagequeue_cnt_add((pq), 1) #define vm_pagequeue_cnt_dec(pq) vm_pagequeue_cnt_add((pq), -1) -static inline u_int -vm_pagequeue_freecnt_adj(int domain, int adj) -{ +void vm_domain_set(int domain); - vm_pagequeue_free_assert_locked(domain); - return (vm_dom[domain].vmd_free_count += adj); -} - /* * vm_pagequeue_domain: * @@ -182,6 +178,9 @@ vm_paging_needed(struct vm_domain *vmd, u_int free_cou return (free_count < vmd->vmd_pageout_wakeup_thresh); } +/* + * Returns TRUE if the domain is below the min paging target. + */ static inline int vm_paging_min(struct vm_domain *vmd) { @@ -190,6 +189,16 @@ vm_paging_min(struct vm_domain *vmd) } /* + * Returns TRUE if the domain is below the severe paging target. + */ +static inline int +vm_paging_severe(struct vm_domain *vmd) +{ + + return (vmd->vmd_free_severe > vmd->vmd_free_count); +} + +/* * Return the number of pages we need to launder. * A positive number indicates that we have a shortfall of clean pages. */ @@ -199,6 +208,23 @@ vm_laundry_target(struct vm_domain *vmd) return (vm_paging_target(vmd)); } + +static inline u_int +vm_pagequeue_freecnt_adj(int domain, int adj) +{ + struct vm_domain *vmd; + u_int ret; + + vm_pagequeue_free_assert_locked(domain); + vmd = VM_DOMAIN(domain); + ret = vmd->vmd_free_count += adj; + if ((!vmd->vmd_minset && vm_paging_min(vmd)) || + (!vmd->vmd_severeset && vm_paging_severe(vmd))) + vm_domain_set(domain); + + return (ret); +} + #endif /* _KERNEL */ #endif /* !_VM_PAGEQUEUE_ */ Modified: user/jeff/numa/sys/vm/vm_swapout.c ============================================================================== --- user/jeff/numa/sys/vm/vm_swapout.c Fri Jan 19 20:33:47 2018 (r328171) +++ user/jeff/numa/sys/vm/vm_swapout.c Fri Jan 19 21:19:57 2018 (r328172) @@ -650,7 +650,7 @@ swapper(void) loop: if (vm_page_count_min()) { - VM_WAIT; + vm_wait_min(); goto loop; }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201801192119.w0JLJver013842>