Date: Sat, 9 Mar 2013 17:25:31 +0000 (UTC) From: Attilio Rao <attilio@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r248110 - in user/attilio/vmcontention/sys: conf fs/tmpfs i386/xen kern vm Message-ID: <201303091725.r29HPV4h040120@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: attilio Date: Sat Mar 9 17:25:31 2013 New Revision: 248110 URL: http://svnweb.freebsd.org/changeset/base/248110 Log: Merge back vmc-playground into vmcontention. vm_radix.{c, h} and _vm_radix.h are copied straight from the branch to preserve history. Added: user/attilio/vmcontention/sys/vm/_vm_radix.h - copied unchanged from r248107, user/attilio/vmc-playground/sys/vm/_vm_radix.h Replaced: user/attilio/vmcontention/sys/vm/vm_radix.c - copied unchanged from r248107, user/attilio/vmc-playground/sys/vm/vm_radix.c user/attilio/vmcontention/sys/vm/vm_radix.h - copied unchanged from r248107, user/attilio/vmc-playground/sys/vm/vm_radix.h Modified: user/attilio/vmcontention/sys/conf/files user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c user/attilio/vmcontention/sys/i386/xen/pmap.c user/attilio/vmcontention/sys/kern/subr_uio.c user/attilio/vmcontention/sys/kern/uipc_shm.c user/attilio/vmcontention/sys/vm/device_pager.c user/attilio/vmcontention/sys/vm/sg_pager.c user/attilio/vmcontention/sys/vm/vm_fault.c user/attilio/vmcontention/sys/vm/vm_init.c user/attilio/vmcontention/sys/vm/vm_mmap.c user/attilio/vmcontention/sys/vm/vm_object.c user/attilio/vmcontention/sys/vm/vm_object.h user/attilio/vmcontention/sys/vm/vm_page.c user/attilio/vmcontention/sys/vm/vm_page.h user/attilio/vmcontention/sys/vm/vm_reserv.c user/attilio/vmcontention/sys/vm/vnode_pager.c Modified: user/attilio/vmcontention/sys/conf/files ============================================================================== --- user/attilio/vmcontention/sys/conf/files Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/conf/files Sat Mar 9 17:25:31 2013 (r248110) @@ -3630,7 +3630,7 @@ vm/vm_page.c standard vm/vm_pageout.c standard vm/vm_pager.c standard vm/vm_phys.c standard -vm/vm_radix.c standard +vm/vm_radix.c standard vm/vm_reserv.c standard vm/vm_unix.c standard vm/vm_zeroidle.c standard Modified: user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c ============================================================================== --- user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_fifoops.c Sat Mar 9 17:25:31 2013 (r248110) @@ -39,7 +39,6 @@ #include <sys/param.h> #include <sys/filedesc.h> #include <sys/proc.h> -#include <sys/systm.h> #include <sys/vnode.h> #include <vm/vm.h> Modified: user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c ============================================================================== --- user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/fs/tmpfs/tmpfs_vnops.c Sat Mar 9 17:25:31 2013 (r248110) @@ -638,34 +638,19 @@ tmpfs_mappedwrite(vm_object_t vobj, vm_o VM_OBJECT_WLOCK(vobj); lookupvpg: - vpg = vm_radix_lookup(&vobj->rtree, idx, VM_RADIX_ANY); - if (vpg != NULL) { - if (vm_page_is_valid(vpg, offset, tlen)) { - if ((vpg->oflags & VPO_BUSY) != 0) { - /* - * Reference the page before unlocking and - * sleeping so that the page daemon is less - * likely to reclaim it. - */ - vm_page_reference(vpg); - vm_page_sleep(vpg, "tmfsmw"); - goto lookupvpg; - } - vm_page_busy(vpg); - vm_page_undirty(vpg); - VM_OBJECT_UNLOCK(vobj); - error = uiomove_fromphys(&vpg, offset, tlen, uio); - } else { - if (vpg->flags & PG_CACHED) { - mtx_lock(&vm_page_queue_free_mtx); - if (vpg->object == vobj) - vm_page_cache_free(vpg); - mtx_unlock(&vm_page_queue_free_mtx); - } - VM_OBJECT_UNLOCK(vobj); - vpg = NULL; + if (((vpg = vm_page_lookup(vobj, idx)) != NULL) && + vm_page_is_valid(vpg, offset, tlen)) { + if ((vpg->oflags & VPO_BUSY) != 0) { + /* + * Reference the page before unlocking and sleeping so + * that the page daemon is less likely to reclaim it. + */ + vm_page_reference(vpg); + vm_page_sleep(vpg, "tmfsmw"); + goto lookupvpg; } - } else + vm_page_busy(vpg); + vm_page_undirty(vpg); VM_OBJECT_WUNLOCK(vobj); error = uiomove_fromphys(&vpg, offset, tlen, uio); } else { Modified: user/attilio/vmcontention/sys/i386/xen/pmap.c ============================================================================== --- user/attilio/vmcontention/sys/i386/xen/pmap.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/i386/xen/pmap.c Sat Mar 9 17:25:31 2013 (r248110) @@ -1335,7 +1335,8 @@ pmap_free_zero_pages(vm_page_t free) while (free != NULL) { m = free; - free = m->right; + free = (void *)m->object; + m->object = NULL; vm_page_free_zero(m); } } @@ -1393,7 +1394,7 @@ _pmap_unwire_ptp(pmap_t pmap, vm_page_t * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ - m->right = *free; + m->object = (void *)*free; *free = m; } @@ -2090,7 +2091,7 @@ out: } if (m_pc == NULL && pv_vafree != 0 && free != NULL) { m_pc = free; - free = m_pc->right; + free = (void *)m_pc->object; /* Recycle a freed page table page. */ m_pc->wire_count = 1; atomic_add_int(&cnt.v_wire_count, 1); Modified: user/attilio/vmcontention/sys/kern/subr_uio.c ============================================================================== --- user/attilio/vmcontention/sys/kern/subr_uio.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/kern/subr_uio.c Sat Mar 9 17:25:31 2013 (r248110) @@ -85,7 +85,6 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k vm_map_entry_t entry; vm_pindex_t upindex; vm_prot_t prot; - vm_page_bits_t vbits; boolean_t wired; KASSERT((uaddr & PAGE_MASK) == 0, @@ -96,7 +95,6 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k * unwired in sf_buf_mext(). */ kern_pg = PHYS_TO_VM_PAGE(vtophys(kaddr)); - vbits = kern_pg->valid; kern_pg->valid = VM_PAGE_BITS_ALL; KASSERT(kern_pg->queue == PQ_NONE && kern_pg->wire_count == 1, ("vm_pgmoveco: kern_pg is not correctly wired")); @@ -107,13 +105,6 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t k return(EFAULT); } VM_OBJECT_WLOCK(uobject); - if (vm_page_insert(kern_pg, uobject, upindex) != 0) { - kern_pg->valid = vbits; - VM_OBJECT_UNLOCK(uobject); - vm_map_lookup_done(map, entry); - return(ENOMEM); - } - vm_page_dirty(kern_pg); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) @@ -131,6 +122,8 @@ retry: if (uobject->backing_object != NULL) pmap_remove(map->pmap, uaddr, uaddr + PAGE_SIZE); } + vm_page_insert(kern_pg, uobject, upindex); + vm_page_dirty(kern_pg); VM_OBJECT_WUNLOCK(uobject); vm_map_lookup_done(map, entry); return(KERN_SUCCESS); Modified: user/attilio/vmcontention/sys/kern/uipc_shm.c ============================================================================== --- user/attilio/vmcontention/sys/kern/uipc_shm.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/kern/uipc_shm.c Sat Mar 9 17:25:31 2013 (r248110) @@ -279,8 +279,7 @@ shm_dotruncate(struct shmfd *shmfd, off_ if (base != 0) { idx = OFF_TO_IDX(length); retry: - m = vm_radix_lookup(&object->rtree, idx, - VM_RADIX_BLACK); + m = vm_page_lookup(object, idx); if (m != NULL) { if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) { Copied: user/attilio/vmcontention/sys/vm/_vm_radix.h (from r248107, user/attilio/vmc-playground/sys/vm/_vm_radix.h) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/attilio/vmcontention/sys/vm/_vm_radix.h Sat Mar 9 17:25:31 2013 (r248110, copy of r248107, user/attilio/vmc-playground/sys/vm/_vm_radix.h) @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2011 Jeffrey Roberson <jeff@freebsd.org> + * Copyright (c) 2008 Mayur Shardul <mayur.shardul@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#ifndef __VM_RADIX_H_ +#define __VM_RADIX_H_ + +/* + * Radix tree root. + */ +struct vm_radix { + uintptr_t rt_root; +}; + +#endif /* !__VM_RADIX_H_ */ Modified: user/attilio/vmcontention/sys/vm/device_pager.c ============================================================================== --- user/attilio/vmcontention/sys/vm/device_pager.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/device_pager.c Sat Mar 9 17:25:31 2013 (r248110) @@ -55,8 +55,6 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_phys.h> #include <vm/uma.h> -#include <machine/cpu.h> - static void dev_pager_init(void); static vm_object_t dev_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred *); @@ -302,7 +300,7 @@ old_dev_pager_fault(vm_object_t object, struct file *fpop; struct thread *td; vm_memattr_t memattr; - int i, ref, ret; + int ref, ret; pidx = OFF_TO_IDX(offset); memattr = object->memattr; @@ -354,10 +352,7 @@ old_dev_pager_fault(vm_object_t object, vm_page_free(*mres); vm_page_unlock(*mres); *mres = page; - while (vm_page_insert(page, object, pidx) != 0) { - for (i = 0; i < 10000000; i++) - cpu_spinwait(); - } + vm_page_insert(page, object, pidx); } page->valid = VM_PAGE_BITS_ALL; return (VM_PAGER_OK); Modified: user/attilio/vmcontention/sys/vm/sg_pager.c ============================================================================== --- user/attilio/vmcontention/sys/vm/sg_pager.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/sg_pager.c Sat Mar 9 17:25:31 2013 (r248110) @@ -182,10 +182,6 @@ sg_pager_getpages(vm_object_t object, vm /* Construct a new fake page. */ page = vm_page_getfake(paddr, memattr); VM_OBJECT_WLOCK(object); - if (vm_page_insert(page, object, offset) != 0) { - vm_page_putfake(page); - return (VM_PAGER_FAIL); - } TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq); /* Free the original pages and insert this fake page into the object. */ @@ -194,6 +190,7 @@ sg_pager_getpages(vm_object_t object, vm vm_page_free(m[i]); vm_page_unlock(m[i]); } + vm_page_insert(page, object, offset); m[reqpage] = page; page->valid = VM_PAGE_BITS_ALL; Modified: user/attilio/vmcontention/sys/vm/vm_fault.c ============================================================================== --- user/attilio/vmcontention/sys/vm/vm_fault.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/vm_fault.c Sat Mar 9 17:25:31 2013 (r248110) @@ -741,7 +741,9 @@ vnode_locked: * process'es object. The page is * automatically made dirty. */ + vm_page_lock(fs.m); vm_page_rename(fs.m, fs.first_object, fs.first_pindex); + vm_page_unlock(fs.m); vm_page_busy(fs.m); fs.first_m = fs.m; fs.m = NULL; Modified: user/attilio/vmcontention/sys/vm/vm_init.c ============================================================================== --- user/attilio/vmcontention/sys/vm/vm_init.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/vm_init.c Sat Mar 9 17:25:31 2013 (r248110) @@ -82,7 +82,6 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_kern.h> #include <vm/vm_object.h> #include <vm/vm_page.h> -#include <vm/vm_radix.h> #include <vm/vm_map.h> #include <vm/vm_pager.h> #include <vm/vm_extern.h> @@ -124,7 +123,6 @@ vm_mem_init(dummy) vm_object_init(); vm_map_startup(); kmem_init(virtual_avail, virtual_end); - vm_radix_init(); pmap_init(); vm_pager_init(); } Modified: user/attilio/vmcontention/sys/vm/vm_mmap.c ============================================================================== --- user/attilio/vmcontention/sys/vm/vm_mmap.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/vm_mmap.c Sat Mar 9 17:25:31 2013 (r248110) @@ -83,7 +83,6 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_pageout.h> #include <vm/vm_extern.h> #include <vm/vm_page.h> -#include <vm/vm_radix.h> #include <vm/vnode_pager.h> #ifdef HWPMC_HOOKS @@ -914,15 +913,10 @@ RestartScan: object->type == OBJT_VNODE) { pindex = OFF_TO_IDX(current->offset + (addr - current->start)); - m = vm_radix_lookup(&object->rtree, - pindex, VM_RADIX_ANY); - - /* Lock just for consistency. */ - mtx_lock(&vm_page_queue_free_mtx); - if (m != NULL && - (m->flags & PG_CACHED) != 0) + m = vm_page_lookup(object, pindex); + if (m == NULL && + vm_page_is_cached(object, pindex)) mincoreinfo = MINCORE_INCORE; - mtx_unlock(&vm_page_queue_free_mtx); if (m != NULL && m->valid == 0) m = NULL; if (m != NULL) Modified: user/attilio/vmcontention/sys/vm/vm_object.c ============================================================================== --- user/attilio/vmcontention/sys/vm/vm_object.c Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/vm_object.c Sat Mar 9 17:25:31 2013 (r248110) @@ -94,6 +94,7 @@ __FBSDID("$FreeBSD$"); #include <vm/swap_pager.h> #include <vm/vm_kern.h> #include <vm/vm_extern.h> +#include <vm/vm_radix.h> #include <vm/vm_reserv.h> #include <vm/uma.h> @@ -165,13 +166,10 @@ vm_object_zdtor(void *mem, int size, voi vm_object_t object; object = (vm_object_t)mem; - KASSERT(object->resident_page_count == 0, - ("object %p resident_page_count = %d", - object, object->resident_page_count)); KASSERT(TAILQ_EMPTY(&object->memq), ("object %p has resident pages in its memq", object)); - KASSERT(object->root == NULL, - ("object %p has resident pages in its tree", object)); + KASSERT(object->rtree.rt_root == 0, + ("object %p has resident pages in its trie", object)); #if VM_NRESERVLEVEL > 0 KASSERT(LIST_EMPTY(&object->rvq), ("object %p has reservations", @@ -183,6 +181,9 @@ vm_object_zdtor(void *mem, int size, voi KASSERT(object->paging_in_progress == 0, ("object %p paging_in_progress = %d", object, object->paging_in_progress)); + KASSERT(object->resident_page_count == 0, + ("object %p resident_page_count = %d", + object, object->resident_page_count)); KASSERT(object->shadow_count == 0, ("object %p shadow_count = %d", object, object->shadow_count)); @@ -199,11 +200,11 @@ vm_object_zinit(void *mem, int size, int rw_init_flags(&object->lock, "vm object", RW_DUPOK); /* These are true for any object that has been freed */ - object->root = NULL; + object->rtree.rt_root = 0; object->paging_in_progress = 0; object->resident_page_count = 0; object->shadow_count = 0; - object->cache = NULL; + object->cache.rt_root = 0; return (0); } @@ -295,6 +296,8 @@ vm_object_init(void) NULL, #endif vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM|UMA_ZONE_NOFREE); + + vm_radix_init(); } void @@ -327,7 +330,7 @@ vm_object_set_memattr(vm_object_t object case OBJT_SG: case OBJT_SWAP: case OBJT_VNODE: - if (object->resident_page_count == 0) + if (!TAILQ_EMPTY(&object->memq)) return (KERN_FAILURE); break; case OBJT_DEAD: @@ -673,11 +676,7 @@ vm_object_destroy(vm_object_t object) void vm_object_terminate(vm_object_t object) { - vm_page_t pa[VM_RADIX_STACK]; - vm_page_t p; - vm_pindex_t start; - u_int exhausted; - int n, i; + vm_page_t p, p_next; VM_OBJECT_ASSERT_WLOCKED(object); @@ -722,73 +721,36 @@ vm_object_terminate(vm_object_t object) * from the object. Rather than incrementally removing each page from * the object, the page and object are reset to any empty state. */ - start = 0; - exhausted = 0; - while (exhausted == 0 && (n = vm_radix_lookupn(&object->rtree, start, - 0, VM_RADIX_ANY, (void **)pa, VM_RADIX_STACK, &start, - &exhausted)) != 0) { - for (i = 0; i < n; i++) { - p = pa[i]; - /* - * Another thread may allocate this cached page from - * the queue before we acquire the page queue free - * mtx. - */ - if (p->flags & PG_CACHED) { - mtx_lock(&vm_page_queue_free_mtx); - if (p->object == object) { - p->object = NULL; - p->valid = 0; - /* Clear PG_CACHED and set PG_FREE. */ - p->flags ^= PG_CACHED | PG_FREE; - cnt.v_cache_count--; - cnt.v_free_count++; - } - mtx_unlock(&vm_page_queue_free_mtx); - continue; - } else if (p->object != object) - continue; - KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0, - ("vm_object_terminate: freeing busy page %p", p)); - vm_page_lock(p); - /* - * Optimize the page's removal from the object by - * resetting its "object" field. Specifically, if - * the page is not wired, then the effect of this - * assignment is that vm_page_free()'s call to - * vm_page_remove() will return immediately without - * modifying the page or the object. - * Anyway, the radix tree cannot be accessed anymore - * from within the object, thus all the nodes need - * to be reclaimed later on. - */ - p->object = NULL; - if (p->wire_count == 0) { - vm_page_free(p); - PCPU_INC(cnt.v_pfree); - } - vm_page_unlock(p); + TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) { + KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0, + ("vm_object_terminate: freeing busy page %p", p)); + vm_page_lock(p); + /* + * Optimize the page's removal from the object by resetting + * its "object" field. Specifically, if the page is not + * wired, then the effect of this assignment is that + * vm_page_free()'s call to vm_page_remove() will return + * immediately without modifying the page or the object. + */ + p->object = NULL; + if (p->wire_count == 0) { + vm_page_free(p); + PCPU_INC(cnt.v_pfree); } - if (n < VM_RADIX_STACK) - break; + vm_page_unlock(p); } - vm_radix_reclaim_allnodes(&object->rtree); /* * If the object contained any pages, then reset it to an empty state. * None of the object's fields, including "resident_page_count", were * modified by the preceding loop. */ if (object->resident_page_count != 0) { + vm_radix_reclaim_allnodes(&object->rtree); TAILQ_INIT(&object->memq); object->resident_page_count = 0; if (object->type == OBJT_VNODE) vdrop(object->handle); } - if (object->cached_page_count != 0) { - object->cached_page_count = 0; - if (object->type == OBJT_VNODE) - vdrop(object->handle); - } #if VM_NRESERVLEVEL > 0 if (__predict_false(!LIST_EMPTY(&object->rvq))) @@ -1306,13 +1268,10 @@ vm_object_shadow( void vm_object_split(vm_map_entry_t entry) { - vm_page_t ma[VM_RADIX_STACK]; - vm_page_t m; + vm_page_t m, m_next; vm_object_t orig_object, new_object, source; - vm_pindex_t idx, offidxstart, start; + vm_pindex_t idx, offidxstart; vm_size_t size; - u_int exhausted; - int i, n; orig_object = entry->object.vm_object; if (orig_object->type != OBJT_DEFAULT && orig_object->type != OBJT_SWAP) @@ -1365,65 +1324,46 @@ vm_object_split(vm_map_entry_t entry) ("orig_object->charge < 0")); orig_object->charge -= ptoa(size); } - start = offidxstart; retry: - exhausted = 0; - while (exhausted == 0 && (n = vm_radix_lookupn(&orig_object->rtree, - start, offidxstart + size, VM_RADIX_ANY, (void **)ma, - VM_RADIX_STACK, &start, &exhausted)) != 0) { - for (i = 0; i < n; i++) { - m = ma[i]; - idx = m->pindex - offidxstart; - if (m->flags & PG_CACHED) { - mtx_lock(&vm_page_queue_free_mtx); - if (m->object == orig_object) - vm_page_cache_rename(m, new_object, - idx); - mtx_unlock(&vm_page_queue_free_mtx); - continue; - } else if (m->object != orig_object) - continue; - /* - * We must wait for pending I/O to complete before - * we can rename the page. - * - * We do not have to VM_PROT_NONE the page as mappings - * should not be changed by this operation. - */ - if ((m->oflags & VPO_BUSY) || m->busy) { - start = m->pindex; - VM_OBJECT_UNLOCK(new_object); - m->oflags |= VPO_WANTED; - VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0); - VM_OBJECT_LOCK(new_object); - goto retry; - } + m = vm_page_find_least(orig_object, offidxstart); + for (; m != NULL && (idx = m->pindex - offidxstart) < size; + m = m_next) { + m_next = TAILQ_NEXT(m, listq); + + /* + * We must wait for pending I/O to complete before we can + * rename the page. + * + * We do not have to VM_PROT_NONE the page as mappings should + * not be changed by this operation. + */ + if ((m->oflags & VPO_BUSY) || m->busy) { + VM_OBJECT_WUNLOCK(new_object); + m->oflags |= VPO_WANTED; + VM_OBJECT_SLEEP(orig_object, m, PVM, "spltwt", 0); + VM_OBJECT_WLOCK(new_object); + goto retry; + } #if VM_NRESERVLEVEL > 0 - /* - * If some of the reservation's allocated pages remain - * with the original object, then transferring the - * reservation to the new object is neither - * particularly beneficial nor particularly harmful as - * compared to leaving the reservation with the - * original object. If, however, all of the - * reservation's allocated pages are transferred to - * the new object, then transferring the reservation - * is typically beneficial. Determining which of - * these two cases applies would be more costly than - * unconditionally renaming the reservation. - */ - vm_reserv_rename(m, new_object, orig_object, - offidxstart); + /* + * If some of the reservation's allocated pages remain with + * the original object, then transferring the reservation to + * the new object is neither particularly beneficial nor + * particularly harmful as compared to leaving the reservation + * with the original object. If, however, all of the + * reservation's allocated pages are transferred to the new + * object, then transferring the reservation is typically + * beneficial. Determining which of these two cases applies + * would be more costly than unconditionally renaming the + * reservation. + */ + vm_reserv_rename(m, new_object, orig_object, offidxstart); #endif - vm_page_rename(m, new_object, idx); - /* - * page automatically made dirty by rename and - * cache handled - */ - vm_page_busy(m); - } - if (n < VM_RADIX_STACK) - break; + vm_page_lock(m); + vm_page_rename(m, new_object, idx); + vm_page_unlock(m); + /* page automatically made dirty by rename and cache handled */ + vm_page_busy(m); } if (orig_object->type == OBJT_SWAP) { /* @@ -1431,6 +1371,19 @@ retry: * and new_object's locks are released and reacquired. */ swap_pager_copy(orig_object, new_object, offidxstart, 0); + + /* + * Transfer any cached pages from orig_object to new_object. + * If swap_pager_copy() found swapped out pages within the + * specified range of orig_object, then it changed + * new_object's type to OBJT_SWAP when it transferred those + * pages to new_object. Otherwise, new_object's type + * should still be OBJT_DEFAULT and orig_object should not + * contain any cached pages within the specified range. + */ + if (__predict_false(!vm_object_cache_is_empty(orig_object))) + vm_page_cache_transfer(orig_object, offidxstart, + new_object); } VM_OBJECT_WUNLOCK(orig_object); TAILQ_FOREACH(m, &new_object->memq, listq) @@ -1449,14 +1402,10 @@ retry: static int vm_object_backing_scan(vm_object_t object, int op) { - vm_page_t pa[VM_RADIX_STACK]; + int r = 1; vm_page_t p; vm_object_t backing_object; - vm_pindex_t backing_offset_index, new_pindex; - vm_pindex_t start; - u_int exhausted; - int color, i, n; - int r = 1; + vm_pindex_t backing_offset_index; VM_OBJECT_ASSERT_WLOCKED(object); VM_OBJECT_ASSERT_WLOCKED(object->backing_object); @@ -1484,41 +1433,15 @@ vm_object_backing_scan(vm_object_t objec if (op & OBSC_COLLAPSE_WAIT) { vm_object_set_flag(backing_object, OBJ_DEAD); } - color = VM_RADIX_BLACK; - if (op & OBSC_COLLAPSE_WAIT) - color |= VM_RADIX_RED; + /* * Our scan */ -restart: - start = 0; - i = n = VM_RADIX_STACK; - exhausted = 0; - for (;;) { - if (i == n) { - if (n < VM_RADIX_STACK) - break; - if (exhausted != 0 || - (n = vm_radix_lookupn(&backing_object->rtree, - start, 0, color, (void **)pa, VM_RADIX_STACK, - &start, &exhausted)) == 0) - break; - i = 0; - } - p = pa[i++]; - /* - * Free cached pages. XXX Why? Emulating old behavior here. - */ - if (p->flags & PG_CACHED) { - mtx_lock(&vm_page_queue_free_mtx); - if (p->object == backing_object) - vm_page_cache_free(p); - mtx_unlock(&vm_page_queue_free_mtx); - continue; - } else if (p->object != backing_object) - continue; + p = TAILQ_FIRST(&backing_object->memq); + while (p) { + vm_page_t next = TAILQ_NEXT(p, listq); + vm_pindex_t new_pindex = p->pindex - backing_offset_index; - new_pindex = p->pindex - backing_offset_index; if (op & OBSC_TEST_ALL_SHADOWED) { vm_page_t pp; @@ -1530,9 +1453,13 @@ restart: * note that we do not busy the backing object's * page. */ - if (p->pindex < backing_offset_index || - new_pindex >= object->size) + if ( + p->pindex < backing_offset_index || + new_pindex >= object->size + ) { + p = next; continue; + } /* * See if the parent has the page or if the parent's @@ -1561,9 +1488,12 @@ restart: vm_page_t pp; if (op & OBSC_COLLAPSE_NOWAIT) { - if ((p->oflags & VPO_BUSY) || !p->valid || - p->busy) + if ((p->oflags & VPO_BUSY) || + !p->valid || + p->busy) { + p = next; continue; + } } else if (op & OBSC_COLLAPSE_WAIT) { if ((p->oflags & VPO_BUSY) || p->busy) { VM_OBJECT_WUNLOCK(object); @@ -1579,7 +1509,8 @@ restart: * should not have changed so we * just restart our scan. */ - goto restart; + p = TAILQ_FIRST(&backing_object->memq); + continue; } } @@ -1615,6 +1546,7 @@ restart: else vm_page_remove(p); vm_page_unlock(p); + p = next; continue; } @@ -1634,6 +1566,7 @@ restart: * page before we can (re)lock the parent. * Hence we can get here. */ + p = next; continue; } if ( @@ -1655,6 +1588,7 @@ restart: else vm_page_remove(p); vm_page_unlock(p); + p = next; continue; } @@ -1673,9 +1607,12 @@ restart: * If the page was mapped to a process, it can remain * mapped through the rename. */ + vm_page_lock(p); vm_page_rename(p, object, new_pindex); + vm_page_unlock(p); /* page automatically made dirty by rename */ } + p = next; } return (r); } @@ -1789,6 +1726,13 @@ vm_object_collapse(vm_object_t object) backing_object, object, OFF_TO_IDX(object->backing_object_offset), TRUE); + + /* + * Free any cached pages from backing_object. + */ + if (__predict_false( + !vm_object_cache_is_empty(backing_object))) + vm_page_cache_free(backing_object, 0, 0); } /* * Object now shadows whatever backing_object did. @@ -1909,112 +1853,80 @@ void vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int options) { - struct vnode *vp; - vm_page_t pa[VM_RADIX_STACK]; - vm_page_t p; - u_int exhausted; - int i, n; + vm_page_t p, next; int wirings; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT((object->flags & OBJ_UNMANAGED) == 0 || (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED, ("vm_object_page_remove: illegal options for object %p", object)); - if (object->resident_page_count == 0 && object->cached_page_count == 0) - return; - vp = NULL; + if (object->resident_page_count == 0) + goto skipmemq; vm_object_pip_add(object, 1); -restart: - exhausted = 0; - while (exhausted == 0 && (n = vm_radix_lookupn(&object->rtree, start, - end, VM_RADIX_ANY, (void **)pa, VM_RADIX_STACK, &start, - &exhausted)) != 0) { - for (i = 0; i < n; i++) { - p = pa[i]; - /* - * Another thread may allocate this cached page from - * the queue before we acquire the page queue free - * mtx. - */ - if (p->flags & PG_CACHED) { - mtx_lock(&vm_page_queue_free_mtx); - if (p->object == object) { - vm_page_cache_free(p); - if (object->type == OBJT_VNODE && - object->cached_page_count == 0) - vp = object->handle; - } - mtx_unlock(&vm_page_queue_free_mtx); - continue; - } else if (p->object != object) - continue; - /* - * If the page is wired for any reason besides - * the existence of managed, wired mappings, then - * it cannot be freed. For example, fictitious - * pages, which represent device memory, are - * inherently wired and cannot be freed. They can, - * however, be invalidated if the option - * OBJPR_CLEANONLY is not specified. - */ - vm_page_lock(p); - if ((wirings = p->wire_count) != 0 && - (wirings = pmap_page_wired_mappings(p)) != - p->wire_count) { - if ((options & OBJPR_NOTMAPPED) == 0) { - pmap_remove_all(p); - /* - * Account for removal of wired - * mappings. - */ - if (wirings != 0) - p->wire_count -= wirings; - } - if ((options & OBJPR_CLEANONLY) == 0) { - p->valid = 0; - vm_page_undirty(p); - } - vm_page_unlock(p); - continue; - } - if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) { - start = p->pindex; - goto restart; - } - KASSERT((p->flags & PG_FICTITIOUS) == 0, - ("vm_object_page_remove: page %p is fictitious", - p)); - if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) { - if ((options & OBJPR_NOTMAPPED) == 0) - pmap_remove_write(p); - if (p->dirty) { - vm_page_unlock(p); - continue; - } - } +again: + p = vm_page_find_least(object, start); + + /* + * Here, the variable "p" is either (1) the page with the least pindex + * greater than or equal to the parameter "start" or (2) NULL. + */ + for (; p != NULL && (p->pindex < end || end == 0); p = next) { + next = TAILQ_NEXT(p, listq); + + /* + * If the page is wired for any reason besides the existence + * of managed, wired mappings, then it cannot be freed. For + * example, fictitious pages, which represent device memory, + * are inherently wired and cannot be freed. They can, + * however, be invalidated if the option OBJPR_CLEANONLY is + * not specified. + */ + vm_page_lock(p); + if ((wirings = p->wire_count) != 0 && + (wirings = pmap_page_wired_mappings(p)) != p->wire_count) { if ((options & OBJPR_NOTMAPPED) == 0) { pmap_remove_all(p); /* Account for removal of wired mappings. */ - if (wirings != 0) { - KASSERT(p->wire_count == wirings, - ("inconsistent wire count %d %d %p", - p->wire_count, wirings, p)); - p->wire_count = 0; - atomic_subtract_int(&cnt.v_wire_count, - 1); - } if (wirings != 0) p->wire_count -= wirings; } - vm_page_free(p); + if ((options & OBJPR_CLEANONLY) == 0) { + p->valid = 0; + vm_page_undirty(p); + } vm_page_unlock(p); + continue; } - if (n < VM_RADIX_STACK) - break; + if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) + goto again; + KASSERT((p->flags & PG_FICTITIOUS) == 0, + ("vm_object_page_remove: page %p is fictitious", p)); + if ((options & OBJPR_CLEANONLY) != 0 && p->valid != 0) { + if ((options & OBJPR_NOTMAPPED) == 0) + pmap_remove_write(p); + if (p->dirty) { + vm_page_unlock(p); + continue; + } + } + if ((options & OBJPR_NOTMAPPED) == 0) { + pmap_remove_all(p); + /* Account for removal of wired mappings. */ + if (wirings != 0) { + KASSERT(p->wire_count == wirings, + ("inconsistent wire count %d %d %p", + p->wire_count, wirings, p)); + p->wire_count = 0; + atomic_subtract_int(&cnt.v_wire_count, 1); + } + } + vm_page_free(p); + vm_page_unlock(p); } vm_object_pip_wakeup(object); - if (vp) - vdrop(vp); +skipmemq: + if (__predict_false(!vm_object_cache_is_empty(object))) + vm_page_cache_free(object, start, end); } /* @@ -2392,9 +2304,8 @@ DB_SHOW_COMMAND(object, vm_object_print_ db_printf(","); count++; - db_printf("(off=0x%jx,page=0x%jx,obj=%p,flags=0x%X)", - (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p), - p->object, p->flags); + db_printf("(off=0x%jx,page=0x%jx)", + (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p)); } if (count != 0) db_printf("\n"); Modified: user/attilio/vmcontention/sys/vm/vm_object.h ============================================================================== --- user/attilio/vmcontention/sys/vm/vm_object.h Sat Mar 9 16:58:19 2013 (r248109) +++ user/attilio/vmcontention/sys/vm/vm_object.h Sat Mar 9 17:25:31 2013 (r248110) @@ -72,7 +72,7 @@ #include <sys/_mutex.h> #include <sys/_rwlock.h> -#include <vm/vm_radix.h> +#include <vm/_vm_radix.h> /* * Types defined: @@ -81,10 +81,10 @@ * * The root of cached pages pool is protected by both the per-object lock * and the free pages queue mutex. - * On insert in the cache splay tree, the per-object lock is expected + * On insert in the cache radix trie, the per-object lock is expected * to be already held and the free pages queue mutex will be * acquired during the operation too. - * On remove and lookup from the cache splay tree, only the free + * On remove and lookup from the cache radix trie, only the free * pages queue mutex is expected to be locked. * These rules allow for reliably checking for the presence of cached * pages with only the per-object lock held, thereby reducing contention @@ -103,7 +103,7 @@ struct vm_object { LIST_HEAD(, vm_object) shadow_head; /* objects that this is a shadow for */ LIST_ENTRY(vm_object) shadow_list; /* chain of shadow objects */ TAILQ_HEAD(, vm_page) memq; /* list of resident pages */ - struct vm_radix rtree; /* root of the resident page radix index tree */ + struct vm_radix rtree; /* root of the resident page radix trie*/ vm_pindex_t size; /* Object size */ int generation; /* generation ID */ int ref_count; /* How many refs?? */ @@ -114,11 +114,11 @@ struct vm_object { u_short pg_color; /* (c) color of first page in obj */ u_int paging_in_progress; /* Paging (in or out) so don't collapse or destroy */ int resident_page_count; /* number of resident pages */ - int cached_page_count; /* number of cached pages */ struct vm_object *backing_object; /* object that I'm a shadow of */ vm_ooffset_t backing_object_offset;/* Offset in backing object */ TAILQ_ENTRY(vm_object) pager_object_list; /* list of all objects of this pager type */ LIST_HEAD(, vm_reserv) rvq; /* list of reservations */ + struct vm_radix cache; /* (o + f) root of the cache page radix trie */ void *handle; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201303091725.r29HPV4h040120>