Date: Thu, 8 Dec 2016 11:35:53 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r309711 - head/sys/vm Message-ID: <201612081135.uB8BZrdw020931@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Thu Dec 8 11:35:53 2016 New Revision: 309711 URL: https://svnweb.freebsd.org/changeset/base/309711 Log: Implement the populate() pager method for phys pager. It allows to provide configurable agressive prefaulting and useful hints to page daemon about memory allocations, on faults for pages managed by phys pager. In fact, this implementation is superior to the MAP_SHARED_PHYS hack from my Postgresql paper, while giving similar benefits of reducing the page faults numbers on SysV shared memory mappings. Reviewed by: alc Tested by: pho Sponsored by: The FreeBSD Foundation MFC after: 3 weeks Modified: head/sys/vm/phys_pager.c Modified: head/sys/vm/phys_pager.c ============================================================================== --- head/sys/vm/phys_pager.c Thu Dec 8 11:26:11 2016 (r309710) +++ head/sys/vm/phys_pager.c Thu Dec 8 11:35:53 2016 (r309711) @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_param.h> #include <vm/vm_object.h> #include <vm/vm_page.h> +#include <vm/vm_pageout.h> #include <vm/vm_pager.h> /* list of phys pager objects */ @@ -98,6 +99,7 @@ phys_pager_alloc(void *handle, vm_ooffse object = object1; object1 = NULL; object->handle = handle; + vm_object_set_flag(object, OBJ_POPULATE); TAILQ_INSERT_TAIL(&phys_pager_object_list, object, pager_object_list); } @@ -109,6 +111,7 @@ phys_pager_alloc(void *handle, vm_ooffse vm_object_deallocate(object1); } else { object = vm_object_allocate(OBJT_PHYS, pindex); + vm_object_set_flag(object, OBJ_POPULATE); } return (object); @@ -157,32 +160,101 @@ phys_pager_getpages(vm_object_t object, return (VM_PAGER_OK); } -static void -phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, - int *rtvals) -{ - - panic("phys_pager_putpage called"); -} - /* * Implement a pretty aggressive clustered getpages strategy. Hint that * everything in an entire 4MB window should be prefaulted at once. * - * XXX 4MB (1024 slots per page table page) is convenient for x86, + * 4MB (1024 slots per page table page) is convenient for x86, * but may not be for other arches. */ #ifndef PHYSCLUSTER #define PHYSCLUSTER 1024 #endif +static int phys_pager_cluster = PHYSCLUSTER; +SYSCTL_INT(_vm, OID_AUTO, phys_pager_cluster, CTLFLAG_RWTUN, + &phys_pager_cluster, 0, + "prefault window size for phys pager"); + +/* + * Max hint to vm_page_alloc() about the further allocation needs + * inside the phys_pager_populate() loop. The number of bits used to + * implement VM_ALLOC_COUNT() determines the hard limit on this value. + * That limit is currently 65535. + */ +#define PHYSALLOC 16 + +static int +phys_pager_populate(vm_object_t object, vm_pindex_t pidx, + int fault_type __unused, vm_prot_t max_prot __unused, vm_pindex_t *first, + vm_pindex_t *last) +{ + vm_page_t m; + vm_pindex_t base, end, i; + int ahead; + + base = rounddown(pidx, phys_pager_cluster); + end = base + phys_pager_cluster - 1; + if (end >= object->size) + end = object->size - 1; + if (*first > base) + base = *first; + if (end > *last) + end = *last; + *first = base; + *last = end; + + for (i = base; i <= end; i++) { +retry: + m = vm_page_lookup(object, i); + if (m == NULL) { + ahead = MIN(end - i, PHYSALLOC); + m = vm_page_alloc(object, i, VM_ALLOC_NORMAL | + VM_ALLOC_ZERO | VM_ALLOC_COUNT(ahead)); + if (m == NULL) { + VM_OBJECT_WUNLOCK(object); + VM_WAIT; + VM_OBJECT_WLOCK(object); + goto retry; + } + if ((m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } else if (vm_page_xbusied(m)) { + vm_page_lock(m); + VM_OBJECT_WUNLOCK(object); + vm_page_busy_sleep(m, "physb", true); + VM_OBJECT_WLOCK(object); + goto retry; + } else { + vm_page_xbusy(m); + if (m->valid != VM_PAGE_BITS_ALL) + vm_page_zero_invalid(m, TRUE); + } + + KASSERT(m->valid == VM_PAGE_BITS_ALL, + ("phys_pager_populate: partially valid page %p", m)); + KASSERT(m->dirty == 0, + ("phys_pager_populate: dirty page %p", m)); + } + return (VM_PAGER_OK); +} + +static void +phys_pager_putpages(vm_object_t object, vm_page_t *m, int count, boolean_t sync, + int *rtvals) +{ + + panic("phys_pager_putpage called"); +} + static boolean_t phys_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after) { vm_pindex_t base, end; - base = rounddown2(pindex, PHYSCLUSTER); - end = base + (PHYSCLUSTER - 1); + base = rounddown(pindex, phys_pager_cluster); + end = base + phys_pager_cluster - 1; if (before != NULL) *before = pindex - base; if (after != NULL) @@ -197,4 +269,5 @@ struct pagerops physpagerops = { .pgo_getpages = phys_pager_getpages, .pgo_putpages = phys_pager_putpages, .pgo_haspage = phys_pager_haspage, + .pgo_populate = phys_pager_populate, };
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201612081135.uB8BZrdw020931>