Date: Sun, 24 Nov 2019 19:12:23 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r355062 - in head/sys: kern vm Message-ID: <201911241912.xAOJCNuc090605@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Sun Nov 24 19:12:23 2019 New Revision: 355062 URL: https://svnweb.freebsd.org/changeset/base/355062 Log: Record part of the owner struct thread pointer into busy_lock. Record as much bits from curthread into busy_lock as fits. Low bits for struct thread * representation are zero due to struct and zone alignment, and they leave space for busy flags (perhaps except statically allocated thread0). Upper bits are not very interesting for assert, and in most practical situations recorded value should allow to manually identify the owner with certainity. Assert that unbusy is performed by the owner, except few places where unbusy is done in io completion handler. For this case, add _unchecked variants of asserts and unbusy primitives. Reviewed by: markj (previous version) Tested by: pho Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D22298 Modified: head/sys/kern/kern_sendfile.c head/sys/vm/vm_page.c head/sys/vm/vm_page.h Modified: head/sys/kern/kern_sendfile.c ============================================================================== --- head/sys/kern/kern_sendfile.c Sun Nov 24 19:06:38 2019 (r355061) +++ head/sys/kern/kern_sendfile.c Sun Nov 24 19:12:23 2019 (r355062) @@ -261,7 +261,7 @@ sendfile_iodone(void *arg, vm_page_t *pg, int count, i for (int i = 0; i < count; i++) if (pg[i] != bogus_page) - vm_page_xunbusy(pg[i]); + vm_page_xunbusy_unchecked(pg[i]); if (error) sfio->error = error; Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Sun Nov 24 19:06:38 2019 (r355061) +++ head/sys/vm/vm_page.c Sun Nov 24 19:12:23 2019 (r355062) @@ -437,7 +437,7 @@ vm_page_init_marker(vm_page_t marker, int queue, uint1 bzero(marker, sizeof(*marker)); marker->flags = PG_MARKER; marker->aflags = aflags; - marker->busy_lock = VPB_SINGLE_EXCLUSIVER; + marker->busy_lock = VPB_CURTHREAD_EXCLUSIVE; marker->queue = queue; } @@ -939,18 +939,19 @@ vm_page_busy_downgrade(vm_page_t m) int vm_page_busy_tryupgrade(vm_page_t m) { - u_int x; + u_int ce, x; vm_page_assert_sbusied(m); x = m->busy_lock; + ce = VPB_CURTHREAD_EXCLUSIVE; for (;;) { if (VPB_SHARERS(x) > 1) return (0); KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1), ("vm_page_busy_tryupgrade: invalid lock state")); if (!atomic_fcmpset_acq_int(&m->busy_lock, &x, - VPB_SINGLE_EXCLUSIVER | (x & VPB_BIT_WAITERS))) + ce | (x & VPB_BIT_WAITERS))) continue; return (1); } @@ -1108,7 +1109,7 @@ vm_page_tryxbusy(vm_page_t m) vm_object_t obj; if (atomic_cmpset_acq_int(&(m)->busy_lock, VPB_UNBUSIED, - VPB_SINGLE_EXCLUSIVER) == 0) + VPB_CURTHREAD_EXCLUSIVE) == 0) return (0); obj = m->object; @@ -1119,6 +1120,14 @@ vm_page_tryxbusy(vm_page_t m) return (1); } +static void +vm_page_xunbusy_hard_tail(vm_page_t m) +{ + atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED); + /* Wake the waiter. */ + wakeup(m); +} + /* * vm_page_xunbusy_hard: * @@ -1127,14 +1136,15 @@ vm_page_tryxbusy(vm_page_t m) void vm_page_xunbusy_hard(vm_page_t m) { - vm_page_assert_xbusied(m); + vm_page_xunbusy_hard_tail(m); +} - /* - * Wake the waiter. - */ - atomic_store_rel_int(&m->busy_lock, VPB_UNBUSIED); - wakeup(m); +void +vm_page_xunbusy_hard_unchecked(vm_page_t m) +{ + vm_page_assert_xbusied_unchecked(m); + vm_page_xunbusy_hard_tail(m); } /* @@ -1228,7 +1238,7 @@ vm_page_initfake(vm_page_t m, vm_paddr_t paddr, vm_mem m->flags = PG_FICTITIOUS; /* Fictitious pages don't use "order" or "pool". */ m->oflags = VPO_UNMANAGED; - m->busy_lock = VPB_SINGLE_EXCLUSIVER; + m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; /* Fictitious pages are unevictable. */ m->ref_count = 1; pmap_page_init(m); @@ -1318,7 +1328,7 @@ vm_page_readahead_finish(vm_page_t m) else vm_page_deactivate(m); vm_page_unlock(m); - vm_page_xunbusy(m); + vm_page_xunbusy_unchecked(m); } /* @@ -1967,7 +1977,7 @@ found: VPO_UNMANAGED : 0; m->busy_lock = VPB_UNBUSIED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) - m->busy_lock = VPB_SINGLE_EXCLUSIVER; + m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); if (req & VM_ALLOC_WIRED) { @@ -2161,7 +2171,7 @@ found: VPO_UNMANAGED : 0; busy_lock = VPB_UNBUSIED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) - busy_lock = VPB_SINGLE_EXCLUSIVER; + busy_lock = VPB_CURTHREAD_EXCLUSIVE; if ((req & VM_ALLOC_SBUSY) != 0) busy_lock = VPB_SHARERS_WORD(1); if ((req & VM_ALLOC_WIRED) != 0) Modified: head/sys/vm/vm_page.h ============================================================================== --- head/sys/vm/vm_page.h Sun Nov 24 19:06:38 2019 (r355061) +++ head/sys/vm/vm_page.h Sun Nov 24 19:12:23 2019 (r355062) @@ -307,7 +307,13 @@ struct vm_page { #define VPB_SHARERS_WORD(x) ((x) << VPB_SHARERS_SHIFT | VPB_BIT_SHARED) #define VPB_ONE_SHARER (1 << VPB_SHARERS_SHIFT) -#define VPB_SINGLE_EXCLUSIVER VPB_BIT_EXCLUSIVE +#define VPB_SINGLE_EXCLUSIVE VPB_BIT_EXCLUSIVE +#ifdef INVARIANTS +#define VPB_CURTHREAD_EXCLUSIVE \ + (VPB_BIT_EXCLUSIVE | ((u_int)(uintptr_t)curthread & ~VPB_BIT_FLAGMASK)) +#else +#define VPB_CURTHREAD_EXCLUSIVE VPB_SINGLE_EXCLUSIVE +#endif #define VPB_UNBUSIED VPB_SHARERS_WORD(0) @@ -649,6 +655,7 @@ void vm_page_updatefake(vm_page_t m, vm_paddr_t paddr, void vm_page_wire(vm_page_t); bool vm_page_wire_mapped(vm_page_t m); void vm_page_xunbusy_hard(vm_page_t m); +void vm_page_xunbusy_hard_unchecked(vm_page_t m); void vm_page_set_validclean (vm_page_t, int, int); void vm_page_clear_dirty(vm_page_t, int, int); void vm_page_set_invalid(vm_page_t, int, int); @@ -684,10 +691,19 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const ("vm_page_assert_unbusied: page %p busy @ %s:%d", \ (m), __FILE__, __LINE__)) -#define vm_page_assert_xbusied(m) \ +#define vm_page_assert_xbusied_unchecked(m) do { \ KASSERT(vm_page_xbusied(m), \ ("vm_page_assert_xbusied: page %p not exclusive busy @ %s:%d", \ - (m), __FILE__, __LINE__)) + (m), __FILE__, __LINE__)); \ +} while (0) +#define vm_page_assert_xbusied(m) do { \ + vm_page_assert_xbusied_unchecked(m); \ + KASSERT((m->busy_lock & ~VPB_BIT_WAITERS) == \ + VPB_CURTHREAD_EXCLUSIVE, \ + ("vm_page_assert_xbusied: page %p busy_lock %#x not owned" \ + " by me @ %s:%d", \ + (m), (m)->busy_lock, __FILE__, __LINE__)); \ +} while (0) #define vm_page_busied(m) \ ((m)->busy_lock != VPB_UNBUSIED) @@ -699,7 +715,7 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const } while (0) #define vm_page_xbusied(m) \ - (((m)->busy_lock & VPB_SINGLE_EXCLUSIVER) != 0) + (((m)->busy_lock & VPB_SINGLE_EXCLUSIVE) != 0) #define vm_page_xbusy(m) do { \ if (!vm_page_tryxbusy(m)) \ @@ -710,8 +726,13 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const /* Note: page m's lock must not be owned by the caller. */ #define vm_page_xunbusy(m) do { \ if (!atomic_cmpset_rel_int(&(m)->busy_lock, \ - VPB_SINGLE_EXCLUSIVER, VPB_UNBUSIED)) \ + VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \ vm_page_xunbusy_hard(m); \ +} while (0) +#define vm_page_xunbusy_unchecked(m) do { \ + if (!atomic_cmpset_rel_int(&(m)->busy_lock, \ + VPB_CURTHREAD_EXCLUSIVE, VPB_UNBUSIED)) \ + vm_page_xunbusy_hard_unchecked(m); \ } while (0) #ifdef INVARIANTS
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201911241912.xAOJCNuc090605>