Date: Fri, 7 Jul 2017 16:59:14 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r320789 - stable/11/sys/vm Message-ID: <201707071659.v67GxEbZ006499@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Fri Jul 7 16:59:14 2017 New Revision: 320789 URL: https://svnweb.freebsd.org/changeset/base/320789 Log: MFC r312208, r312994: Optimize vm_object_madvise(). Modified: stable/11/sys/vm/vm_object.c stable/11/sys/vm/vm_page.c Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/vm/vm_object.c ============================================================================== --- stable/11/sys/vm/vm_object.c Fri Jul 7 16:58:40 2017 (r320788) +++ stable/11/sys/vm/vm_object.c Fri Jul 7 16:59:14 2017 (r320789) @@ -1075,6 +1075,33 @@ vm_object_sync(vm_object_t object, vm_ooffset_t offset } /* + * Determine whether the given advice can be applied to the object. Advice is + * not applied to unmanaged pages since they never belong to page queues, and + * since MADV_FREE is destructive, it can apply only to anonymous pages that + * have been mapped at most once. + */ +static bool +vm_object_advice_applies(vm_object_t object, int advice) +{ + + if ((object->flags & OBJ_UNMANAGED) != 0) + return (false); + if (advice != MADV_FREE) + return (true); + return ((object->type == OBJT_DEFAULT || object->type == OBJT_SWAP) && + (object->flags & OBJ_ONEMAPPING) != 0); +} + +static void +vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, + vm_size_t size) +{ + + if (advice == MADV_FREE && object->type == OBJT_SWAP) + swap_pager_freespace(object, pindex, size); +} + +/* * vm_object_madvise: * * Implements the madvise function at the object/page level. @@ -1097,96 +1124,109 @@ vm_object_sync(vm_object_t object, vm_ooffset_t offset */ void vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end, - int advise) + int advice) { vm_pindex_t tpindex; vm_object_t backing_object, tobject; - vm_page_t m; + vm_page_t m, tm; if (object == NULL) return; - VM_OBJECT_WLOCK(object); - /* - * Locate and adjust resident pages - */ - for (; pindex < end; pindex += 1) { + relookup: + VM_OBJECT_WLOCK(object); + if (!vm_object_advice_applies(object, advice)) { + VM_OBJECT_WUNLOCK(object); + return; + } + for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) { tobject = object; - tpindex = pindex; -shadowlookup: + /* - * MADV_FREE only operates on OBJT_DEFAULT or OBJT_SWAP pages - * and those pages must be OBJ_ONEMAPPING. + * If the next page isn't resident in the top-level object, we + * need to search the shadow chain. When applying MADV_FREE, we + * take care to release any swap space used to store + * non-resident pages. */ - if (advise == MADV_FREE) { - if ((tobject->type != OBJT_DEFAULT && - tobject->type != OBJT_SWAP) || - (tobject->flags & OBJ_ONEMAPPING) == 0) { - goto unlock_tobject; - } - } else if ((tobject->flags & OBJ_UNMANAGED) != 0) - goto unlock_tobject; - m = vm_page_lookup(tobject, tpindex); - if (m == NULL) { + if (m == NULL || pindex < m->pindex) { /* - * There may be swap even if there is no backing page + * Optimize a common case: if the top-level object has + * no backing object, we can skip over the non-resident + * range in constant time. */ - if (advise == MADV_FREE && tobject->type == OBJT_SWAP) - swap_pager_freespace(tobject, tpindex, 1); - /* - * next object - */ - backing_object = tobject->backing_object; - if (backing_object == NULL) - goto unlock_tobject; - VM_OBJECT_WLOCK(backing_object); - tpindex += OFF_TO_IDX(tobject->backing_object_offset); - if (tobject != object) - VM_OBJECT_WUNLOCK(tobject); - tobject = backing_object; - goto shadowlookup; - } else if (m->valid != VM_PAGE_BITS_ALL) - goto unlock_tobject; + if (object->backing_object == NULL) { + tpindex = (m != NULL && m->pindex < end) ? + m->pindex : end; + vm_object_madvise_freespace(object, advice, + pindex, tpindex - pindex); + if ((pindex = tpindex) == end) + break; + goto next_page; + } + + tpindex = pindex; + do { + vm_object_madvise_freespace(tobject, advice, + tpindex, 1); + /* + * Prepare to search the next object in the + * chain. + */ + backing_object = tobject->backing_object; + if (backing_object == NULL) + goto next_pindex; + VM_OBJECT_WLOCK(backing_object); + tpindex += + OFF_TO_IDX(tobject->backing_object_offset); + if (tobject != object) + VM_OBJECT_WUNLOCK(tobject); + tobject = backing_object; + if (!vm_object_advice_applies(tobject, advice)) + goto next_pindex; + } while ((tm = vm_page_lookup(tobject, tpindex)) == + NULL); + } else { +next_page: + tm = m; + m = TAILQ_NEXT(m, listq); + } + /* * If the page is not in a normal state, skip it. */ - vm_page_lock(m); - if (m->hold_count != 0 || m->wire_count != 0) { - vm_page_unlock(m); - goto unlock_tobject; + if (tm->valid != VM_PAGE_BITS_ALL) + goto next_pindex; + vm_page_lock(tm); + if (tm->hold_count != 0 || tm->wire_count != 0) { + vm_page_unlock(tm); + goto next_pindex; } - KASSERT((m->flags & PG_FICTITIOUS) == 0, - ("vm_object_madvise: page %p is fictitious", m)); - KASSERT((m->oflags & VPO_UNMANAGED) == 0, - ("vm_object_madvise: page %p is not managed", m)); - if (vm_page_busied(m)) { - if (advise == MADV_WILLNEED) { + KASSERT((tm->flags & PG_FICTITIOUS) == 0, + ("vm_object_madvise: page %p is fictitious", tm)); + KASSERT((tm->oflags & VPO_UNMANAGED) == 0, + ("vm_object_madvise: page %p is not managed", tm)); + if (vm_page_busied(tm)) { + if (object != tobject) + VM_OBJECT_WUNLOCK(tobject); + VM_OBJECT_WUNLOCK(object); + if (advice == MADV_WILLNEED) { /* * Reference the page before unlocking and * sleeping so that the page daemon is less - * likely to reclaim it. + * likely to reclaim it. */ - vm_page_aflag_set(m, PGA_REFERENCED); + vm_page_aflag_set(tm, PGA_REFERENCED); } - if (object != tobject) - VM_OBJECT_WUNLOCK(object); - VM_OBJECT_WUNLOCK(tobject); - vm_page_busy_sleep(m, "madvpo", false); - VM_OBJECT_WLOCK(object); + vm_page_busy_sleep(tm, "madvpo", false); goto relookup; } - if (advise == MADV_WILLNEED) { - vm_page_activate(m); - } else { - vm_page_advise(m, advise); - } - vm_page_unlock(m); - if (advise == MADV_FREE && tobject->type == OBJT_SWAP) - swap_pager_freespace(tobject, tpindex, 1); -unlock_tobject: + vm_page_advise(tm, advice); + vm_page_unlock(tm); + vm_object_madvise_freespace(tobject, advice, tm->pindex, 1); +next_pindex: if (tobject != object) VM_OBJECT_WUNLOCK(tobject); - } + } VM_OBJECT_WUNLOCK(object); } Modified: stable/11/sys/vm/vm_page.c ============================================================================== --- stable/11/sys/vm/vm_page.c Fri Jul 7 16:58:40 2017 (r320788) +++ stable/11/sys/vm/vm_page.c Fri Jul 7 16:59:14 2017 (r320789) @@ -3027,7 +3027,7 @@ vm_page_try_to_free(vm_page_t m) /* * vm_page_advise * - * Deactivate or do nothing, as appropriate. + * Apply the specified advice to the given page. * * The object and page must be locked. */ @@ -3045,8 +3045,11 @@ vm_page_advise(vm_page_t m, int advice) * would result in a page fault on a later access. */ vm_page_undirty(m); - else if (advice != MADV_DONTNEED) + else if (advice != MADV_DONTNEED) { + if (advice == MADV_WILLNEED) + vm_page_activate(m); return; + } /* * Clear any references to the page. Otherwise, the page daemon will
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201707071659.v67GxEbZ006499>