Date: Tue, 25 Jun 2013 22:46:52 +0000 (UTC) From: Attilio Rao <attilio@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r252220 - in user/attilio/vmobj-readlock/sys: cddl/contrib/opensolaris/uts/common/fs/zfs dev/drm2/i915 dev/drm2/ttm fs/tmpfs kern vm Message-ID: <201306252246.r5PMkqeS082296@svn.freebsd.org>
index | next in thread | raw e-mail
Author: attilio Date: Tue Jun 25 22:46:52 2013 New Revision: 252220 URL: http://svnweb.freebsd.org/changeset/base/252220 Log: Reimplement the busy mechanism by using atomics and not depending by object lock for the operation themselves. This way the busy can be performed also within a path where the obj is locked in read way. vm_page_sleep() is renamed into vm_page_busy_sleep() in order to show it is a busy specific operation. Sponsored by: EMC / Isilon storage division Modified: user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c user/attilio/vmobj-readlock/sys/kern/vfs_bio.c user/attilio/vmobj-readlock/sys/vm/vm_fault.c user/attilio/vmobj-readlock/sys/vm/vm_object.c user/attilio/vmobj-readlock/sys/vm/vm_page.c user/attilio/vmobj-readlock/sys/vm/vm_page.h user/attilio/vmobj-readlock/sys/vm/vm_pageout.c user/attilio/vmobj-readlock/sys/vm/vm_phys.c user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Modified: user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue Jun 25 22:46:52 2013 (r252220) @@ -345,7 +345,7 @@ page_busy(vnode_t *vp, int64_t start, in vm_page_reference(pp); vm_page_lock(pp); zfs_vmobject_wunlock(obj); - vm_page_sleep(pp, "zfsmwb"); + vm_page_busy_sleep(pp, "zfsmwb"); zfs_vmobject_wlock(obj); continue; } @@ -429,7 +429,7 @@ update_pages(vnode_t *vp, int64_t start, ("zfs update_pages: unaligned data in putpages case")); KASSERT(pp->valid == VM_PAGE_BITS_ALL, ("zfs update_pages: invalid page in putpages case")); - KASSERT(pp->busy > 0, + KASSERT(vm_page_busy_rlocked(pp), ("zfs update_pages: unbusy page in putpages case")); KASSERT(!pmap_page_is_write_mapped(pp), ("zfs update_pages: writable page in putpages case")); Modified: user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c ============================================================================== --- user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/dev/drm2/i915/i915_gem.c Tue Jun 25 22:46:52 2013 (r252220) @@ -1382,7 +1382,7 @@ retry: DRM_UNLOCK(dev); vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); - vm_page_sleep(m, "915pee"); + vm_page_busy_sleep(m, "915pee"); goto retry; } goto have_page; @@ -1440,7 +1440,7 @@ retry: DRM_UNLOCK(dev); vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); - vm_page_sleep(m, "915pbs"); + vm_page_busy_sleep(m, "915pbs"); goto retry; } m->valid = VM_PAGE_BITS_ALL; Modified: user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c ============================================================================== --- user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/dev/drm2/ttm/ttm_bo_vm.c Tue Jun 25 22:46:52 2013 (r252220) @@ -215,7 +215,7 @@ reserve: if (vm_page_busy_locked(m)) { vm_page_lock(m); VM_OBJECT_WUNLOCK(vm_obj); - vm_page_sleep(m, "ttmpbs"); + vm_page_busy_sleep(m, "ttmpbs"); VM_OBJECT_WLOCK(vm_obj); ttm_mem_io_unlock(man); ttm_bo_unreserve(bo); Modified: user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c ============================================================================== --- user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/fs/tmpfs/tmpfs_vnops.c Tue Jun 25 22:46:52 2013 (r252220) @@ -479,8 +479,7 @@ tmpfs_nocacheread(vm_object_t tobj, vm_p } else vm_page_zero_invalid(m, TRUE); } - vm_page_busy_wunlock(m); - vm_page_busy_rlock(m); + vm_page_busy_downgrade(m); VM_OBJECT_WUNLOCK(tobj); error = uiomove_fromphys(&m, offset, tlen, uio); VM_OBJECT_WLOCK(tobj); @@ -594,8 +593,7 @@ tmpfs_mappedwrite(vm_object_t tobj, size } else vm_page_zero_invalid(tpg, TRUE); } - vm_page_busy_wunlock(tpg); - vm_page_busy_rlock(tpg); + vm_page_busy_downgrade(tpg); VM_OBJECT_WUNLOCK(tobj); error = uiomove_fromphys(&tpg, offset, tlen, uio); VM_OBJECT_WLOCK(tobj); Modified: user/attilio/vmobj-readlock/sys/kern/vfs_bio.c ============================================================================== --- user/attilio/vmobj-readlock/sys/kern/vfs_bio.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/kern/vfs_bio.c Tue Jun 25 22:46:52 2013 (r252220) @@ -4003,7 +4003,7 @@ vfs_drain_busy_pages(struct buf *bp) while (vm_page_busy_wlocked(m)) { vm_page_lock(m); VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); - vm_page_sleep(m, "vbpage"); + vm_page_busy_sleep(m, "vbpage"); VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); } } @@ -4269,7 +4269,7 @@ vm_hold_free_pages(struct buf *bp, int n for (index = newnpages; index < bp->b_npages; index++) { p = bp->b_pages[index]; bp->b_pages[index] = NULL; - if (p->busy != 0) + if (vm_page_busy_rlocked(p)) printf("vm_hold_free_pages: blkno: %jd, lblkno: %jd\n", (intmax_t)bp->b_blkno, (intmax_t)bp->b_lblkno); p->wire_count--; Modified: user/attilio/vmobj-readlock/sys/vm/vm_fault.c ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vm_fault.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vm_fault.c Tue Jun 25 22:46:52 2013 (r252220) @@ -926,12 +926,13 @@ vnode_locked: if (m_hold != NULL) { *m_hold = fs.m; if (fault_flags & VM_FAULT_IOBUSY) - vm_page_busy_rlock(fs.m); + vm_page_busy_downgrade(fs.m); else vm_page_hold(fs.m); } vm_page_unlock(fs.m); - vm_page_busy_wunlock(fs.m); + if (m_hold == NULL || (fault_flags & VM_FAULT_IOBUSY) == 0) + vm_page_busy_wunlock(fs.m); /* * Unlock everything, and return Modified: user/attilio/vmobj-readlock/sys/vm/vm_object.c ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vm_object.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vm_object.c Tue Jun 25 22:46:52 2013 (r252220) @@ -1167,7 +1167,7 @@ shadowlookup: if (object != tobject) VM_OBJECT_WUNLOCK(object); VM_OBJECT_WUNLOCK(tobject); - vm_page_sleep(m, "madvpo"); + vm_page_busy_sleep(m, "madvpo"); VM_OBJECT_WLOCK(object); goto relookup; } @@ -1346,7 +1346,7 @@ retry: VM_OBJECT_WUNLOCK(new_object); vm_page_lock(m); VM_OBJECT_WUNLOCK(orig_object); - vm_page_sleep(m, "spltwt"); + vm_page_busy_sleep(m, "spltwt"); VM_OBJECT_WLOCK(orig_object); VM_OBJECT_WLOCK(new_object); goto retry; @@ -1505,7 +1505,7 @@ vm_object_backing_scan(vm_object_t objec VM_OBJECT_WUNLOCK(object); vm_page_lock(p); VM_OBJECT_WUNLOCK(backing_object); - vm_page_sleep(p, "vmocol"); + vm_page_busy_sleep(p, "vmocol"); VM_OBJECT_WLOCK(object); VM_OBJECT_WLOCK(backing_object); /* @@ -1905,7 +1905,7 @@ again: } if (vm_page_busy_locked(p)) { VM_OBJECT_WUNLOCK(object); - vm_page_sleep(p, "vmopar"); + vm_page_busy_sleep(p, "vmopar"); VM_OBJECT_WLOCK(object); goto again; } Modified: user/attilio/vmobj-readlock/sys/vm/vm_page.c ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vm_page.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vm_page.c Tue Jun 25 22:46:52 2013 (r252220) @@ -470,86 +470,123 @@ vm_page_reference(vm_page_t m) } void -vm_page_busy_assert_unlocked(vm_page_t m) +vm_page_busy_downgrade(vm_page_t m) { + u_int retry, x; - KASSERT((m->oflags & VPO_BUSY) == 0 && m->busy == 0, - ("vm_page_busy_assert_unlocked: page %p busy", (void *)m)); -} + vm_page_busy_assert_wlocked(m); -void -vm_page_busy_assert_wlocked(vm_page_t m) -{ - - KASSERT((m->oflags & VPO_BUSY) != 0, - ("vm_page_busy_assert_wlocked: page %p not write busy", (void *)m)); + retry = 0; + for (;;) { + if (retry++ > 0) + panic("vm_page_busy_downgrade: failed loop %p", m); + x = m->busy_lock; + x &= VPB_LOCK_WAITERS; + if (atomic_cmpset_rel_int(&m->busy_lock, + VPB_SINGLE_WRITER | x, VPB_READERS_LOCK(1) | x)) + break; + } } int -vm_page_busy_locked(vm_page_t m) +vm_page_busy_rlocked(vm_page_t m) { + u_int x; - return ((m->oflags & VPO_BUSY) != 0 || m->busy != 0); + x = m->busy_lock; + return ((x & VPB_LOCK_READ) != 0 && x != VPB_UNLOCKED); } void vm_page_busy_runlock(vm_page_t m) { + u_int retry, x; + + vm_page_busy_assert_rlocked(m); + + retry = 0; + for (;;) { + if (retry++ > 10000) + panic("vm_page_busy_runlock: failed loop %p", m); + x = m->busy_lock; + if (VPB_READERS(x) > 1) { + if (atomic_cmpset_int(&m->busy_lock, x, + x - VPB_ONE_READER)) + break; + continue; + } + if ((x & VPB_LOCK_WAITERS) == 0) { + KASSERT(x == VPB_READERS_LOCK(1), + ("vm_page_busy_runlock: invalid lock state")); + if (atomic_cmpset_int(&m->busy_lock, + VPB_READERS_LOCK(1), VPB_UNLOCKED)) + break; + continue; + } + KASSERT(x == (VPB_READERS_LOCK(1) | VPB_LOCK_WAITERS), + ("vm_page_busy_runlock: invalid lock state for waiters")); - VM_OBJECT_ASSERT_WLOCKED(m->object); - KASSERT(m->busy > 0, - ("vm_page_busy_runlock: page %p is not read busy", m)); - m->busy--; - if (m->busy == 0) { vm_page_lock(m); - vm_page_flash(m); + if (atomic_cmpset_int(&m->busy_lock, x, VPB_UNLOCKED)) { + vm_page_unlock(m); + continue; + } + wakeup(m); vm_page_unlock(m); + break; } } -int -vm_page_busy_tryrlock(vm_page_t m) +/* + * vm_page_busy_sleep: + * + * Sleep and release the page lock, using the page pointer as wchan. + * This is used to implement the hard-path of busying mechanism. + * + * The given page must be locked. + */ +void +vm_page_busy_sleep(vm_page_t m, const char *wmesg) { + u_int x; - VM_OBJECT_ASSERT_WLOCKED(m->object); - if ((m->oflags & VPO_BUSY) != 0) - return (0); - m->busy++; - return (1); -} - -int -vm_page_busy_trywlock(vm_page_t m) -{ + vm_page_lock_assert(m, MA_OWNED); - VM_OBJECT_ASSERT_WLOCKED(m->object); - if ((m->oflags & VPO_BUSY) != 0 || m->busy != 0) - return (0); - m->oflags |= VPO_BUSY; - return (1); + x = m->busy_lock; + if (x == VPB_UNLOCKED) { + vm_page_unlock(m); + return; + } + if ((x & VPB_LOCK_WAITERS) == 0 && + !atomic_cmpset_int(&m->busy_lock, x, x | VPB_LOCK_WAITERS)) { + vm_page_unlock(m); + return; + } + msleep(m, vm_page_lockptr(m), PVM | PDROP, wmesg, 0); } int -vm_page_busy_wlocked(vm_page_t m) +vm_page_busy_tryrlock(vm_page_t m) { + u_int x; - return ((m->oflags & VPO_BUSY) != 0); + x = m->busy_lock; + return ((x & VPB_LOCK_READ) != 0 && + atomic_cmpset_acq_int(&m->busy_lock, x, x + VPB_ONE_READER)); } void -vm_page_busy_wunlock(vm_page_t m) +vm_page_busy_wunlock_hard(vm_page_t m) { - VM_OBJECT_ASSERT_WLOCKED(m->object); - KASSERT(m->oflags & VPO_BUSY, - ("vm_page_busy_wunlock: page not busy")); - m->oflags &= ~VPO_BUSY; + vm_page_busy_assert_wlocked(m); + vm_page_lock(m); - vm_page_flash(m); + atomic_store_rel_int(&m->busy_lock, VPB_UNLOCKED); + wakeup(m); vm_page_unlock(m); } - /* * vm_page_flash: * @@ -558,12 +595,22 @@ vm_page_busy_wunlock(vm_page_t m) void vm_page_flash(vm_page_t m) { + u_int retry, x; vm_page_lock_assert(m, MA_OWNED); - if (m->flags & PG_WANTED) { - m->flags &= ~PG_WANTED; - wakeup(m); + + retry = 0; + for (;;) { + if (retry++ > 1000) + panic("vm_page_flash: failed loop %p", m); + x = m->busy_lock; + if ((x & VPB_LOCK_WAITERS) == 0) + return; + if (atomic_cmpset_int(&m->busy_lock, x, + x & (~VPB_LOCK_WAITERS))) + break; } + wakeup(m); } /* @@ -678,7 +725,8 @@ vm_page_initfake(vm_page_t m, vm_paddr_t /* Fictitious pages don't use "segind". */ m->flags = PG_FICTITIOUS; /* Fictitious pages don't use "order" or "pool". */ - m->oflags = VPO_UNMANAGED | VPO_BUSY; + m->oflags = VPO_UNMANAGED; + m->busy_lock = VPB_SINGLE_WRITER; m->wire_count = 1; memattr: pmap_page_set_memattr(m, memattr); @@ -758,7 +806,7 @@ vm_page_readahead_finish(vm_page_t m) * unless the page is wanted by another thread. */ vm_page_lock(m); - if (m->flags & PG_WANTED) + if ((m->busy_lock & VPB_LOCK_WAITERS) != 0) vm_page_activate(m); else vm_page_deactivate(m); @@ -778,22 +826,6 @@ vm_page_readahead_finish(vm_page_t m) } /* - * _vm_page_sleep_onpage: - * - * Sleep and release the page lock, using the page pointer as wchan. - * - * The given page must be locked. - */ -static inline int -_vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo) -{ - - vm_page_lock_assert(m, MA_OWNED); - m->flags |= PG_WANTED; - return (msleep(m, vm_page_lockptr(m), pri | PDROP, wmesg, timo)); -} - -/* * vm_page_sleep_if_busy: * * Sleep and release the page queues lock if the page is busied. @@ -821,7 +853,7 @@ vm_page_sleep_if_busy(vm_page_t m, const obj = m->object; vm_page_lock(m); VM_OBJECT_WUNLOCK(obj); - _vm_page_sleep_onpage(m, PVM, msg, 0); + vm_page_busy_sleep(m, msg); VM_OBJECT_WLOCK(obj); return (TRUE); } @@ -829,20 +861,6 @@ vm_page_sleep_if_busy(vm_page_t m, const } /* - * vm_page_sleep_onpage: - * - * External version of _vm_page_sleep_onpage(). - * - * Check the inline version for comments. - */ -int -vm_page_sleep_onpage(vm_page_t m, int pri, const char *wmesg, int timo) -{ - - return (_vm_page_sleep_onpage(m, pri, wmesg, timo)); -} - -/* * vm_page_dirty_KBI: [ internal use only ] * * Set all bits in the page's dirty field. @@ -972,7 +990,6 @@ vm_page_remove(vm_page_t m) return; VM_OBJECT_ASSERT_WLOCKED(object); if (vm_page_busy_wlocked(m)) { - m->oflags &= ~VPO_BUSY; lockacq = FALSE; if ((m->oflags & VPO_UNMANAGED) != 0 && !mtx_owned(vm_page_lockptr(m))) { @@ -980,6 +997,7 @@ vm_page_remove(vm_page_t m) vm_page_lock(m); } vm_page_flash(m); + atomic_store_rel_int(&m->busy_lock, VPB_UNLOCKED); if (lockacq) vm_page_unlock(m); } @@ -1370,7 +1388,8 @@ vm_page_alloc(vm_object_t object, vm_pin ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue)); KASSERT(m->wire_count == 0, ("vm_page_alloc: page %p is wired", m)); KASSERT(m->hold_count == 0, ("vm_page_alloc: page %p is held", m)); - KASSERT(m->busy == 0, ("vm_page_alloc: page %p is busy", m)); + KASSERT(!vm_page_busy_rlocked(m), + ("vm_page_alloc: page %p is busy", m)); KASSERT(m->dirty == 0, ("vm_page_alloc: page %p is dirty", m)); KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("vm_page_alloc: page %p has unexpected memattr %d", m, @@ -1414,10 +1433,11 @@ vm_page_alloc(vm_object_t object, vm_pin m->aflags = 0; m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; + m->busy_lock = VPB_UNLOCKED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_RBUSY)) == 0) - m->oflags |= VPO_BUSY; + m->busy_lock = VPB_SINGLE_WRITER; if ((req & VM_ALLOC_RBUSY) != 0) - m->busy++; + m->busy_lock = VPB_READERS_LOCK(1); if (req & VM_ALLOC_WIRED) { /* * The page lock is not required for wiring a page until that @@ -1585,8 +1605,6 @@ retry: atomic_add_int(&cnt.v_wire_count, npages); oflags = VPO_UNMANAGED; if (object != NULL) { - if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0) - oflags |= VPO_BUSY; if (object->memattr != VM_MEMATTR_DEFAULT && memattr == VM_MEMATTR_DEFAULT) memattr = object->memattr; @@ -1594,8 +1612,13 @@ retry: for (m = m_ret; m < &m_ret[npages]; m++) { m->aflags = 0; m->flags = (m->flags | PG_NODUMP) & flags; - if ((req & VM_ALLOC_RBUSY) != 0) - m->busy++; + m->busy_lock = VPB_UNLOCKED; + if (object != NULL) { + if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_RBUSY)) == 0) + m->busy_lock = VPB_SINGLE_WRITER; + if ((req & VM_ALLOC_RBUSY) != 0) + m->busy_lock = VPB_READERS_LOCK(1); + } if ((req & VM_ALLOC_WIRED) != 0) m->wire_count = 1; /* Unmanaged pages don't use "act_count". */ @@ -1638,7 +1661,7 @@ vm_page_alloc_init(vm_page_t m) ("vm_page_alloc_init: page %p is wired", m)); KASSERT(m->hold_count == 0, ("vm_page_alloc_init: page %p is held", m)); - KASSERT(m->busy == 0, + KASSERT(!vm_page_busy_rlocked(m), ("vm_page_alloc_init: page %p is busy", m)); KASSERT(m->dirty == 0, ("vm_page_alloc_init: page %p is dirty", m)); @@ -1997,7 +2020,7 @@ vm_page_free_toq(vm_page_t m) if (VM_PAGE_IS_FREE(m)) panic("vm_page_free: freeing free page %p", m); - else if (m->busy != 0) + else if (vm_page_busy_rlocked(m)) panic("vm_page_free: freeing busy page %p", m); /* @@ -2443,6 +2466,7 @@ vm_page_t vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) { vm_page_t m; + int sleep; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT((allocflags & VM_ALLOC_RETRY) != 0, @@ -2452,8 +2476,9 @@ vm_page_grab(vm_object_t object, vm_pind ("vm_page_grab: VM_ALLOC_RBUSY/VM_ALLOC_IGN_RBUSY mismatch")); retrylookup: if ((m = vm_page_lookup(object, pindex)) != NULL) { - if ((m->oflags & VPO_BUSY) != 0 || - ((allocflags & VM_ALLOC_IGN_RBUSY) == 0 && m->busy != 0)) { + sleep = (allocflags & VM_ALLOC_IGN_RBUSY) != 0 ? + vm_page_busy_wlocked(m) : vm_page_busy_locked(m); + if (sleep) { /* * Reference the page before unlocking and * sleeping so that the page daemon is less @@ -2462,7 +2487,7 @@ retrylookup: vm_page_aflag_set(m, PGA_REFERENCED); vm_page_lock(m); VM_OBJECT_WUNLOCK(object); - vm_page_sleep(m, "pgrbwt"); + vm_page_busy_sleep(m, "pgrbwt"); VM_OBJECT_WLOCK(object); goto retrylookup; } else { @@ -3020,9 +3045,9 @@ DB_SHOW_COMMAND(pginfo, vm_page_print_pg m = (vm_page_t)addr; db_printf( "page %p obj %p pidx 0x%jx phys 0x%jx q %d hold %d wire %d\n" - " af 0x%x of 0x%x f 0x%x act %d busy %d valid 0x%x dirty 0x%x\n", + " af 0x%x of 0x%x f 0x%x act %d busy %x valid 0x%x dirty 0x%x\n", m, m->object, (uintmax_t)m->pindex, (uintmax_t)m->phys_addr, m->queue, m->hold_count, m->wire_count, m->aflags, m->oflags, - m->flags, m->act_count, m->busy, m->valid, m->dirty); + m->flags, m->act_count, m->busy_lock, m->valid, m->dirty); } #endif /* DDB */ Modified: user/attilio/vmobj-readlock/sys/vm/vm_page.h ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vm_page.h Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vm_page.h Tue Jun 25 22:46:52 2013 (r252220) @@ -155,11 +155,12 @@ struct vm_page { uint8_t oflags; /* page VPO_* flags (O) */ uint16_t flags; /* page PG_* flags (P) */ u_char act_count; /* page usage count (P) */ - u_char busy; /* page busy count (O) */ + u_char basy; /* page busy count (O) */ /* NOTE that these must support one bit per DEV_BSIZE in a page!!! */ /* so, on normal X86 kernels, they must be at least 8 bits wide */ vm_page_bits_t valid; /* map of valid DEV_BSIZE chunks (O) */ vm_page_bits_t dirty; /* map of dirty DEV_BSIZE chunks (M) */ + u_int busy_lock; /* busy owners lock */ }; /* @@ -176,12 +177,31 @@ struct vm_page { * mappings, and such pages are also not on any PQ queue. * */ -#define VPO_BUSY 0x01 /* page is in transit */ +#define VPO_UNUSED01 0x01 /* --available-- */ #define VPO_SWAPSLEEP 0x02 /* waiting for swap to finish */ #define VPO_UNMANAGED 0x04 /* no PV management for page */ #define VPO_SWAPINPROG 0x08 /* swap I/O in progress on page */ #define VPO_NOSYNC 0x10 /* do not collect for syncer */ +/* + * ARXXX: Insert comments for busy here. + */ +#define VPB_LOCK_READ 0x01 +#define VPB_LOCK_WRITE 0x02 +#define VPB_LOCK_WAITERS 0x04 +#define VPB_LOCK_FLAGMASK \ + (VPB_LOCK_READ | VPB_LOCK_WRITE | VPB_LOCK_WAITERS) + +#define VPB_READERS_SHIFT 3 +#define VPB_READERS(x) \ + (((x) & ~VPB_LOCK_FLAGMASK) >> VPB_READERS_SHIFT) +#define VPB_READERS_LOCK(x) ((x) << VPB_READERS_SHIFT | VPB_LOCK_READ) +#define VPB_ONE_READER (1 << VPB_READERS_SHIFT) + +#define VPB_SINGLE_WRITER VPB_LOCK_WRITE + +#define VPB_UNLOCKED VPB_READERS_LOCK(0) + #define PQ_NONE 255 #define PQ_INACTIVE 0 #define PQ_ACTIVE 1 @@ -282,7 +302,6 @@ extern struct mtx_padalign pa_lock[]; #define PG_WINATCFLS 0x0040 /* flush dirty page on inactive q */ #define PG_NODUMP 0x0080 /* don't include this page in a dump */ #define PG_UNHOLDFREE 0x0100 /* delayed free of a held page */ -#define PG_WANTED 0x0200 /* someone is waiting for page */ /* * Misc constants. @@ -372,14 +391,12 @@ malloc2vm_flags(int malloc_flags) } #endif -void vm_page_busy_assert_unlocked(vm_page_t m); -void vm_page_busy_assert_wlocked(vm_page_t m); -int vm_page_busy_locked(vm_page_t m); +void vm_page_busy_downgrade(vm_page_t m); +int vm_page_busy_rlocked(vm_page_t m); void vm_page_busy_runlock(vm_page_t m); +void vm_page_busy_sleep(vm_page_t m, const char *msg); int vm_page_busy_tryrlock(vm_page_t m); -int vm_page_busy_trywlock(vm_page_t m); -int vm_page_busy_wlocked(vm_page_t m); -void vm_page_busy_wunlock(vm_page_t m); +void vm_page_busy_wunlock_hard(vm_page_t m); void vm_page_flash(vm_page_t m); void vm_page_hold(vm_page_t mem); void vm_page_unhold(vm_page_t mem); @@ -420,7 +437,6 @@ void vm_page_requeue(vm_page_t m); void vm_page_requeue_locked(vm_page_t m); void vm_page_set_valid_range(vm_page_t m, int base, int size); int vm_page_sleep_if_busy(vm_page_t m, const char *msg); -int vm_page_sleep_onpage(vm_page_t m, int pri, const char *msg, int timo); vm_offset_t vm_page_startup(vm_offset_t vaddr); void vm_page_unhold_pages(vm_page_t *ma, int count); void vm_page_unwire (vm_page_t, int); @@ -448,16 +464,46 @@ void vm_page_assert_locked_KBI(vm_page_t void vm_page_lock_assert_KBI(vm_page_t m, int a, const char *file, int line); #endif +#define vm_page_busy_assert_rlocked(m) \ + KASSERT(vm_page_busy_rlocked(m), \ + ("vm_page_busy_assert_rlocked: page %p not read busy @ %s:%d", \ + (void *)m, __FILE__, __LINE__)); + +#define vm_page_busy_assert_unlocked(m) \ + KASSERT(!vm_page_busy_locked(m), \ + ("vm_page_busy_assert_wlocked: page %p busy @ %s:%d", \ + (void *)m, __FILE__, __LINE__)); + +#define vm_page_busy_assert_wlocked(m) \ + KASSERT(vm_page_busy_wlocked(m), \ + ("vm_page_busy_assert_wlocked: page %p not write busy @ %s:%d", \ + (void *)m, __FILE__, __LINE__)); + +#define vm_page_busy_locked(m) \ + ((m)->busy_lock != VPB_UNLOCKED) + #define vm_page_busy_rlock(m) do { \ if (!vm_page_busy_tryrlock(m)) \ panic("%s: page %p failed read busing", __func__, m); \ } while (0) +#define vm_page_busy_trywlock(m) \ + (atomic_cmpset_acq_int(&m->busy_lock, VPB_UNLOCKED, VPB_SINGLE_WRITER)) + #define vm_page_busy_wlock(m) do { \ if (!vm_page_busy_trywlock(m)) \ panic("%s: page %p failed write busing", __func__, m); \ } while (0) +#define vm_page_busy_wlocked(m) \ + ((m->busy_lock & VPB_SINGLE_WRITER) != 0) + +#define vm_page_busy_wunlock(m) do { \ + if (!atomic_cmpset_rel_int(&(m)->busy_lock, VPB_SINGLE_WRITER, \ + VPB_UNLOCKED)) \ + vm_page_busy_wunlock_hard(m); \ +} while (0) + #ifdef INVARIANTS void vm_page_object_lock_assert(vm_page_t m); #define VM_PAGE_OBJECT_LOCK_ASSERT(m) vm_page_object_lock_assert(m) @@ -556,21 +602,6 @@ vm_page_dirty(vm_page_t m) } /* - * vm_page_sleep: - * - * Convenience wrapper around vm_page_sleep_onpage(), passing - * PVM priority and 0 timeout values. Unlocks the page upon return. - * - * The page must be locked. - */ -static __inline void -vm_page_sleep(vm_page_t m, const char *msg) -{ - - vm_page_sleep_onpage(m, PVM, msg, 0); -} - -/* * vm_page_remque: * * If the given page is in a page queue, then remove it from that page Modified: user/attilio/vmobj-readlock/sys/vm/vm_pageout.c ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vm_pageout.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vm_pageout.c Tue Jun 25 22:46:52 2013 (r252220) @@ -241,7 +241,7 @@ vm_pageout_init_marker(vm_page_t marker, bzero(marker, sizeof(*marker)); marker->flags = PG_MARKER; - marker->oflags = VPO_BUSY; + marker->busy_lock = VPB_SINGLE_WRITER; marker->queue = queue; marker->hold_count = 1; } Modified: user/attilio/vmobj-readlock/sys/vm/vm_phys.c ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vm_phys.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vm_phys.c Tue Jun 25 22:46:52 2013 (r252220) @@ -561,7 +561,7 @@ vm_phys_fictitious_reg_range(vm_paddr_t vm_page_initfake(&fp[i], start + PAGE_SIZE * i, memattr); pmap_page_init(&fp[i]); fp[i].oflags &= ~VPO_UNMANAGED; - fp[i].oflags &= ~VPO_BUSY; + fp[i].busy_lock = VPB_UNLOCKED; } mtx_lock(&vm_phys_fictitious_reg_mtx); for (segind = 0; segind < VM_PHYS_FICTITIOUS_NSEGS; segind++) { Modified: user/attilio/vmobj-readlock/sys/vm/vnode_pager.c ============================================================================== --- user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Tue Jun 25 22:14:32 2013 (r252219) +++ user/attilio/vmobj-readlock/sys/vm/vnode_pager.c Tue Jun 25 22:46:52 2013 (r252220) @@ -1140,8 +1140,7 @@ vnode_pager_generic_putpages(struct vnod * pmap operation. */ m = ma[ncount - 1]; - KASSERT(m->busy > 0, - ("vnode_pager_generic_putpages: page %p is not busy", m)); + vm_page_busy_assert_rlocked(m); KASSERT(!pmap_page_is_write_mapped(m), ("vnode_pager_generic_putpages: page %p is not read-only", m)); vm_page_clear_dirty(m, pgoff, PAGE_SIZE -help
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201306252246.r5PMkqeS082296>