Date: Tue, 23 May 2017 07:27:30 +0000 (UTC) From: Mark Johnston <markj@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r318716 - in stable/11: sys/amd64/amd64 sys/arm64/arm64 sys/cddl/compat/opensolaris/sys sys/cddl/contrib/opensolaris/uts/common/fs/zfs sys/fs/tmpfs sys/i386/i386 sys/kern sys/sys sys/vm... Message-ID: <201705230727.v4N7RUYV078792@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: markj Date: Tue May 23 07:27:30 2017 New Revision: 318716 URL: https://svnweb.freebsd.org/changeset/base/318716 Log: MFC r308474, r308691, r309203, r309365, r309703, r309898, r310720, r308489, r308706: Add PQ_LAUNDRY and remove PG_CACHED pages. Modified: stable/11/sys/amd64/amd64/pmap.c stable/11/sys/arm64/arm64/pmap.c stable/11/sys/cddl/compat/opensolaris/sys/vnode.h stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c stable/11/sys/fs/tmpfs/tmpfs_subr.c stable/11/sys/i386/i386/pmap.c stable/11/sys/kern/kern_exec.c stable/11/sys/kern/uipc_shm.c stable/11/sys/sys/vmmeter.h stable/11/sys/vm/_vm_radix.h stable/11/sys/vm/swap_pager.c stable/11/sys/vm/vm_fault.c stable/11/sys/vm/vm_map.c stable/11/sys/vm/vm_meter.c stable/11/sys/vm/vm_mmap.c stable/11/sys/vm/vm_object.c stable/11/sys/vm/vm_object.h stable/11/sys/vm/vm_page.c stable/11/sys/vm/vm_page.h stable/11/sys/vm/vm_pageout.c stable/11/sys/vm/vm_phys.c stable/11/sys/vm/vm_radix.c stable/11/sys/vm/vm_radix.h stable/11/sys/vm/vm_reserv.c stable/11/sys/vm/vm_reserv.h stable/11/sys/vm/vnode_pager.c stable/11/usr.bin/systat/systat.1 stable/11/usr.bin/systat/vmstat.c stable/11/usr.bin/top/machine.c stable/11/usr.bin/top/top.local.1 stable/11/usr.bin/vmstat/vmstat.c Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/amd64/amd64/pmap.c ============================================================================== --- stable/11/sys/amd64/amd64/pmap.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/amd64/amd64/pmap.c Tue May 23 07:27:30 2017 (r318716) @@ -614,7 +614,6 @@ static vm_page_t pmap_enter_quick_locked static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); -static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va); static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask); static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, struct rwlock **lockp); @@ -625,7 +624,7 @@ static int pmap_remove_pde(pmap_t pmap, struct spglist *free, struct rwlock **lockp); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); -static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte); +static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, struct spglist *free); static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, @@ -2218,29 +2217,17 @@ pmap_insert_pt_page(pmap_t pmap, vm_page } /* - * Looks for a page table page mapping the specified virtual address in the - * specified pmap's collection of idle page table pages. Returns NULL if there - * is no page table page corresponding to the specified virtual address. + * Removes the page table page mapping the specified virtual address from the + * specified pmap's collection of idle page table pages, and returns it. + * Otherwise, returns NULL if there is no page table page corresponding to the + * specified virtual address. */ static __inline vm_page_t -pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va) +pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); - return (vm_radix_lookup(&pmap->pm_root, pmap_pde_pindex(va))); -} - -/* - * Removes the specified page table page from the specified pmap's collection - * of idle page table pages. The specified page table page must be a member of - * the pmap's collection. - */ -static __inline void -pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte) -{ - - PMAP_LOCK_ASSERT(pmap, MA_OWNED); - vm_radix_remove(&pmap->pm_root, mpte->pindex); + return (vm_radix_remove(&pmap->pm_root, pmap_pde_pindex(va))); } /* @@ -3460,10 +3447,8 @@ pmap_demote_pde_locked(pmap_t pmap, pd_e oldpde = *pde; KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V), ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V")); - if ((oldpde & PG_A) != 0 && (mpte = pmap_lookup_pt_page(pmap, va)) != - NULL) - pmap_remove_pt_page(pmap, mpte); - else { + if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == + NULL) { KASSERT((oldpde & PG_W) == 0, ("pmap_demote_pde: page table page for a wired mapping" " is missing")); @@ -3577,11 +3562,10 @@ pmap_remove_kernel_pde(pmap_t pmap, pd_e KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mpte = pmap_lookup_pt_page(pmap, va); + mpte = pmap_remove_pt_page(pmap, va); if (mpte == NULL) panic("pmap_remove_kernel_pde: Missing pt page."); - pmap_remove_pt_page(pmap, mpte); mptepa = VM_PAGE_TO_PHYS(mpte); newpde = mptepa | X86_PG_M | X86_PG_A | X86_PG_RW | X86_PG_V; @@ -3668,9 +3652,8 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t if (pmap == kernel_pmap) { pmap_remove_kernel_pde(pmap, pdq, sva); } else { - mpte = pmap_lookup_pt_page(pmap, sva); + mpte = pmap_remove_pt_page(pmap, sva); if (mpte != NULL) { - pmap_remove_pt_page(pmap, mpte); pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pde: pte page wire count error")); @@ -5533,9 +5516,8 @@ pmap_remove_pages(pmap_t pmap) TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } - mpte = pmap_lookup_pt_page(pmap, pv->pv_va); + mpte = pmap_remove_pt_page(pmap, pv->pv_va); if (mpte != NULL) { - pmap_remove_pt_page(pmap, mpte); pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pages: pte page wire count error")); Modified: stable/11/sys/arm64/arm64/pmap.c ============================================================================== --- stable/11/sys/arm64/arm64/pmap.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/arm64/arm64/pmap.c Tue May 23 07:27:30 2017 (r318716) @@ -2514,29 +2514,17 @@ pmap_insert_pt_page(pmap_t pmap, vm_page } /* - * Looks for a page table page mapping the specified virtual address in the - * specified pmap's collection of idle page table pages. Returns NULL if there - * is no page table page corresponding to the specified virtual address. + * Removes the page table page mapping the specified virtual address from the + * specified pmap's collection of idle page table pages, and returns it. + * Otherwise, returns NULL if there is no page table page corresponding to the + * specified virtual address. */ static __inline vm_page_t -pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va) +pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); - return (vm_radix_lookup(&pmap->pm_root, pmap_l2_pindex(va))); -} - -/* - * Removes the specified page table page from the specified pmap's collection - * of idle page table pages. The specified page table page must be a member of - * the pmap's collection. - */ -static __inline void -pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte) -{ - - PMAP_LOCK_ASSERT(pmap, MA_OWNED); - vm_radix_remove(&pmap->pm_root, mpte->pindex); + return (vm_radix_remove(&pmap->pm_root, pmap_l2_pindex(va))); } /* @@ -3605,10 +3593,9 @@ pmap_remove_pages(pmap_t pmap) TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } - ml3 = pmap_lookup_pt_page(pmap, + ml3 = pmap_remove_pt_page(pmap, pv->pv_va); if (ml3 != NULL) { - pmap_remove_pt_page(pmap, ml3); pmap_resident_count_dec(pmap,1); KASSERT(ml3->wire_count == NL3PG, ("pmap_remove_pages: l3 page wire count error")); @@ -4381,9 +4368,7 @@ pmap_demote_l2_locked(pmap_t pmap, pt_en return (NULL); } - if ((ml3 = pmap_lookup_pt_page(pmap, va)) != NULL) { - pmap_remove_pt_page(pmap, ml3); - } else { + if ((ml3 = pmap_remove_pt_page(pmap, va)) == NULL) { ml3 = vm_page_alloc(NULL, pmap_l2_pindex(va), (VIRT_IN_DMAP(va) ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); Modified: stable/11/sys/cddl/compat/opensolaris/sys/vnode.h ============================================================================== --- stable/11/sys/cddl/compat/opensolaris/sys/vnode.h Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/cddl/compat/opensolaris/sys/vnode.h Tue May 23 07:27:30 2017 (r318716) @@ -75,8 +75,7 @@ vn_is_readonly(vnode_t *vp) #define vn_mountedvfs(vp) ((vp)->v_mountedhere) #define vn_has_cached_data(vp) \ ((vp)->v_object != NULL && \ - ((vp)->v_object->resident_page_count > 0 || \ - !vm_object_cache_is_empty((vp)->v_object))) + (vp)->v_object->resident_page_count > 0) #define vn_exists(vp) do { } while (0) #define vn_invalid(vp) do { } while (0) #define vn_renamepath(tdvp, svp, tnm, lentnm) do { } while (0) Modified: stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c ============================================================================== --- stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c Tue May 23 07:27:30 2017 (r318716) @@ -426,10 +426,6 @@ page_busy(vnode_t *vp, int64_t start, in continue; } vm_page_sbusy(pp); - } else if (pp == NULL) { - pp = vm_page_alloc(obj, OFF_TO_IDX(start), - VM_ALLOC_SYSTEM | VM_ALLOC_IFCACHED | - VM_ALLOC_SBUSY); } else { ASSERT(pp != NULL && !pp->valid); pp = NULL; Modified: stable/11/sys/fs/tmpfs/tmpfs_subr.c ============================================================================== --- stable/11/sys/fs/tmpfs/tmpfs_subr.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/fs/tmpfs/tmpfs_subr.c Tue May 23 07:27:30 2017 (r318716) @@ -1401,12 +1401,9 @@ retry: VM_WAIT; VM_OBJECT_WLOCK(uobj); goto retry; - } else if (m->valid != VM_PAGE_BITS_ALL) - rv = vm_pager_get_pages(uobj, &m, 1, - NULL, NULL); - else - /* A cached page was reactivated. */ - rv = VM_PAGER_OK; + } + rv = vm_pager_get_pages(uobj, &m, 1, NULL, + NULL); vm_page_lock(m); if (rv == VM_PAGER_OK) { vm_page_deactivate(m); Modified: stable/11/sys/i386/i386/pmap.c ============================================================================== --- stable/11/sys/i386/i386/pmap.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/i386/i386/pmap.c Tue May 23 07:27:30 2017 (r318716) @@ -306,7 +306,6 @@ static boolean_t pmap_is_modified_pvh(st static boolean_t pmap_is_referenced_pvh(struct md_page *pvh); static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); static void pmap_kenter_pde(vm_offset_t va, pd_entry_t newpde); -static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va); static void pmap_pde_attr(pd_entry_t *pde, int cache_bits); static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, @@ -316,7 +315,7 @@ static void pmap_remove_pde(pmap_t pmap, struct spglist *free); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, struct spglist *free); -static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte); +static vm_page_t pmap_remove_pt_page(pmap_t pmap, vm_offset_t va); static void pmap_remove_page(struct pmap *pmap, vm_offset_t va, struct spglist *free); static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, @@ -1727,29 +1726,17 @@ pmap_insert_pt_page(pmap_t pmap, vm_page } /* - * Looks for a page table page mapping the specified virtual address in the - * specified pmap's collection of idle page table pages. Returns NULL if there - * is no page table page corresponding to the specified virtual address. + * Removes the page table page mapping the specified virtual address from the + * specified pmap's collection of idle page table pages, and returns it. + * Otherwise, returns NULL if there is no page table page corresponding to the + * specified virtual address. */ static __inline vm_page_t -pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va) +pmap_remove_pt_page(pmap_t pmap, vm_offset_t va) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); - return (vm_radix_lookup(&pmap->pm_root, va >> PDRSHIFT)); -} - -/* - * Removes the specified page table page from the specified pmap's collection - * of idle page table pages. The specified page table page must be a member of - * the pmap's collection. - */ -static __inline void -pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte) -{ - - PMAP_LOCK_ASSERT(pmap, MA_OWNED); - vm_radix_remove(&pmap->pm_root, mpte->pindex); + return (vm_radix_remove(&pmap->pm_root, va >> PDRSHIFT)); } /* @@ -2645,10 +2632,8 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t oldpde = *pde; KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V), ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V")); - if ((oldpde & PG_A) != 0 && (mpte = pmap_lookup_pt_page(pmap, va)) != - NULL) - pmap_remove_pt_page(pmap, mpte); - else { + if ((oldpde & PG_A) == 0 || (mpte = pmap_remove_pt_page(pmap, va)) == + NULL) { KASSERT((oldpde & PG_W) == 0, ("pmap_demote_pde: page table page for a wired mapping" " is missing")); @@ -2786,11 +2771,10 @@ pmap_remove_kernel_pde(pmap_t pmap, pd_e vm_page_t mpte; PMAP_LOCK_ASSERT(pmap, MA_OWNED); - mpte = pmap_lookup_pt_page(pmap, va); + mpte = pmap_remove_pt_page(pmap, va); if (mpte == NULL) panic("pmap_remove_kernel_pde: Missing pt page."); - pmap_remove_pt_page(pmap, mpte); mptepa = VM_PAGE_TO_PHYS(mpte); newpde = mptepa | PG_M | PG_A | PG_RW | PG_V; @@ -2872,9 +2856,8 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t if (pmap == kernel_pmap) { pmap_remove_kernel_pde(pmap, pdq, sva); } else { - mpte = pmap_lookup_pt_page(pmap, sva); + mpte = pmap_remove_pt_page(pmap, sva); if (mpte != NULL) { - pmap_remove_pt_page(pmap, mpte); pmap->pm_stats.resident_count--; KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pde: pte page wire count error")); @@ -4616,9 +4599,8 @@ pmap_remove_pages(pmap_t pmap) if (TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } - mpte = pmap_lookup_pt_page(pmap, pv->pv_va); + mpte = pmap_remove_pt_page(pmap, pv->pv_va); if (mpte != NULL) { - pmap_remove_pt_page(pmap, mpte); pmap->pm_stats.resident_count--; KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pages: pte page wire count error")); Modified: stable/11/sys/kern/kern_exec.c ============================================================================== --- stable/11/sys/kern/kern_exec.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/kern/kern_exec.c Tue May 23 07:27:30 2017 (r318716) @@ -1006,7 +1006,7 @@ exec_map_first_page(imgp) break; } else { ma[i] = vm_page_alloc(object, i, - VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED); + VM_ALLOC_NORMAL); if (ma[i] == NULL) break; } Modified: stable/11/sys/kern/uipc_shm.c ============================================================================== --- stable/11/sys/kern/uipc_shm.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/kern/uipc_shm.c Tue May 23 07:27:30 2017 (r318716) @@ -455,12 +455,9 @@ retry: VM_WAIT; VM_OBJECT_WLOCK(object); goto retry; - } else if (m->valid != VM_PAGE_BITS_ALL) - rv = vm_pager_get_pages(object, &m, 1, - NULL, NULL); - else - /* A cached page was reactivated. */ - rv = VM_PAGER_OK; + } + rv = vm_pager_get_pages(object, &m, 1, NULL, + NULL); vm_page_lock(m); if (rv == VM_PAGER_OK) { vm_page_deactivate(m); Modified: stable/11/sys/sys/vmmeter.h ============================================================================== --- stable/11/sys/sys/vmmeter.h Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/sys/vmmeter.h Tue May 23 07:27:30 2017 (r318716) @@ -75,9 +75,10 @@ struct vmmeter { u_int v_vnodepgsin; /* (p) vnode_pager pages paged in */ u_int v_vnodepgsout; /* (p) vnode pager pages paged out */ u_int v_intrans; /* (p) intransit blocking page faults */ - u_int v_reactivated; /* (f) pages reactivated from free list */ + u_int v_reactivated; /* (p) pages reactivated by the pagedaemon */ u_int v_pdwakeups; /* (p) times daemon has awaken from sleep */ u_int v_pdpages; /* (p) pages analyzed by daemon */ + u_int v_pdshortfalls; /* (p) page reclamation shortfalls */ u_int v_tcached; /* (p) total pages cached */ u_int v_dfree; /* (p) pages freed by daemon */ @@ -96,6 +97,7 @@ struct vmmeter { u_int v_active_count; /* (q) pages active */ u_int v_inactive_target; /* (c) pages desired inactive */ u_int v_inactive_count; /* (q) pages inactive */ + u_int v_laundry_count; /* (q) pages eligible for laundering */ u_int v_cache_count; /* (f) pages on cache queue */ u_int v_pageout_free_min; /* (c) min pages reserved for kernel */ u_int v_interrupt_free_min; /* (c) reserved pages for int code */ @@ -111,7 +113,6 @@ struct vmmeter { u_int v_vforkpages; /* (p) VM pages affected by vfork() */ u_int v_rforkpages; /* (p) VM pages affected by rfork() */ u_int v_kthreadpages; /* (p) VM pages affected by fork() by kernel */ - u_int v_spare[2]; }; #ifdef _KERNEL @@ -184,6 +185,25 @@ vm_paging_needed(void) (u_int)vm_pageout_wakeup_thresh); } +/* + * Return the number of pages we need to launder. + * A positive number indicates that we have a shortfall of clean pages. + */ +static inline int +vm_laundry_target(void) +{ + + return (vm_paging_target()); +} + +/* + * Obtain the value of a per-CPU counter. + */ +#define VM_METER_PCPU_CNT(member) \ + vm_meter_cnt(__offsetof(struct vmmeter, member)) + +u_int vm_meter_cnt(size_t); + #endif /* systemwide totals computed every five seconds */ Modified: stable/11/sys/vm/_vm_radix.h ============================================================================== --- stable/11/sys/vm/_vm_radix.h Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/_vm_radix.h Tue May 23 07:27:30 2017 (r318716) @@ -36,12 +36,8 @@ */ struct vm_radix { uintptr_t rt_root; - uint8_t rt_flags; }; -#define RT_INSERT_INPROG 0x01 -#define RT_TRIE_MODIFIED 0x02 - #ifdef _KERNEL static __inline boolean_t Modified: stable/11/sys/vm/swap_pager.c ============================================================================== --- stable/11/sys/vm/swap_pager.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/swap_pager.c Tue May 23 07:27:30 2017 (r318716) @@ -1126,7 +1126,7 @@ swap_pager_getpages(vm_object_t object, if (shift != 0) { for (i = 1; i <= shift; i++) { p = vm_page_alloc(object, m[0]->pindex - i, - VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED); + VM_ALLOC_NORMAL); if (p == NULL) { /* Shift allocated pages to the left. */ for (j = 0; j < i - 1; j++) @@ -1144,8 +1144,7 @@ swap_pager_getpages(vm_object_t object, if (rahead != NULL) { for (i = 0; i < *rahead; i++) { p = vm_page_alloc(object, - m[reqcount - 1]->pindex + i + 1, - VM_ALLOC_NORMAL | VM_ALLOC_IFNOTCACHED); + m[reqcount - 1]->pindex + i + 1, VM_ALLOC_NORMAL); if (p == NULL) break; bp->b_pages[shift + reqcount + i] = p; @@ -1549,17 +1548,18 @@ swp_pager_async_iodone(struct buf *bp) * For write success, clear the dirty * status, then finish the I/O ( which decrements the * busy count and possibly wakes waiter's up ). + * A page is only written to swap after a period of + * inactivity. Therefore, we do not expect it to be + * reused. */ KASSERT(!pmap_page_is_write_mapped(m), ("swp_pager_async_iodone: page %p is not write" " protected", m)); vm_page_undirty(m); + vm_page_lock(m); + vm_page_deactivate_noreuse(m); + vm_page_unlock(m); vm_page_sunbusy(m); - if (vm_page_count_severe()) { - vm_page_lock(m); - vm_page_try_to_cache(m); - vm_page_unlock(m); - } } } @@ -1635,12 +1635,15 @@ swap_pager_isswapped(vm_object_t object, /* * SWP_PAGER_FORCE_PAGEIN() - force a swap block to be paged in * - * This routine dissociates the page at the given index within a - * swap block from its backing store, paging it in if necessary. - * If the page is paged in, it is placed in the inactive queue, - * since it had its backing store ripped out from under it. - * We also attempt to swap in all other pages in the swap block, - * we only guarantee that the one at the specified index is + * This routine dissociates the page at the given index within an object + * from its backing store, paging it in if it does not reside in memory. + * If the page is paged in, it is marked dirty and placed in the laundry + * queue. The page is marked dirty because it no longer has backing + * store. It is placed in the laundry queue because it has not been + * accessed recently. Otherwise, it would already reside in memory. + * + * We also attempt to swap in all other pages in the swap block. + * However, we only guarantee that the one at the specified index is * paged in. * * XXX - The code to page the whole block in doesn't work, so we @@ -1669,7 +1672,7 @@ swp_pager_force_pagein(vm_object_t objec vm_object_pip_wakeup(object); vm_page_dirty(m); vm_page_lock(m); - vm_page_deactivate(m); + vm_page_launder(m); vm_page_unlock(m); vm_page_xunbusy(m); vm_pager_page_unswapped(m); Modified: stable/11/sys/vm/vm_fault.c ============================================================================== --- stable/11/sys/vm/vm_fault.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_fault.c Tue May 23 07:27:30 2017 (r318716) @@ -485,11 +485,12 @@ int vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags, vm_page_t *m_hold) { - vm_prot_t prot; - vm_object_t next_object; struct faultstate fs; struct vnode *vp; + vm_object_t next_object, retry_object; vm_offset_t e_end, e_start; + vm_pindex_t retry_pindex; + vm_prot_t prot, retry_prot; int ahead, alloc_req, behind, cluster_offset, error, era, faultcount; int locked, nera, result, rv; u_char behavior; @@ -755,8 +756,7 @@ RetryFault:; unlock_and_deallocate(&fs); VM_WAITPFAULT; goto RetryFault; - } else if (fs.m->valid == VM_PAGE_BITS_ALL) - break; + } } readrest: @@ -1143,10 +1143,6 @@ readrest: * lookup. */ if (!fs.lookup_still_valid) { - vm_object_t retry_object; - vm_pindex_t retry_pindex; - vm_prot_t retry_prot; - if (!vm_map_trylock_read(fs.map)) { release_page(&fs); unlock_and_deallocate(&fs); Modified: stable/11/sys/vm/vm_map.c ============================================================================== --- stable/11/sys/vm/vm_map.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_map.c Tue May 23 07:27:30 2017 (r318716) @@ -1858,9 +1858,7 @@ vm_map_submap( * limited number of page mappings are created at the low-end of the * specified address range. (For this purpose, a superpage mapping * counts as one page mapping.) Otherwise, all resident pages within - * the specified address range are mapped. Because these mappings are - * being created speculatively, cached pages are not reactivated and - * mapped. + * the specified address range are mapped. */ static void vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, Modified: stable/11/sys/vm/vm_meter.c ============================================================================== --- stable/11/sys/vm/vm_meter.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_meter.c Tue May 23 07:27:30 2017 (r318716) @@ -209,29 +209,37 @@ vmtotal(SYSCTL_HANDLER_ARGS) } /* - * vcnt() - accumulate statistics from all cpus and the global cnt - * structure. + * vm_meter_cnt() - accumulate statistics from all cpus and the global cnt + * structure. * * The vmmeter structure is now per-cpu as well as global. Those * statistics which can be kept on a per-cpu basis (to avoid cache * stalls between cpus) can be moved to the per-cpu vmmeter. Remaining * statistics, such as v_free_reserved, are left in the global * structure. - * - * (sysctl_oid *oidp, void *arg1, int arg2, struct sysctl_req *req) */ -static int -vcnt(SYSCTL_HANDLER_ARGS) +u_int +vm_meter_cnt(size_t offset) { - int count = *(int *)arg1; - int offset = (char *)arg1 - (char *)&vm_cnt; + struct pcpu *pcpu; + u_int count; int i; + count = *(u_int *)((char *)&vm_cnt + offset); CPU_FOREACH(i) { - struct pcpu *pcpu = pcpu_find(i); - count += *(int *)((char *)&pcpu->pc_cnt + offset); + pcpu = pcpu_find(i); + count += *(u_int *)((char *)&pcpu->pc_cnt + offset); } - return (SYSCTL_OUT(req, &count, sizeof(int))); + return (count); +} + +static int +cnt_sysctl(SYSCTL_HANDLER_ARGS) +{ + u_int count; + + count = vm_meter_cnt((char *)arg1 - (char *)&vm_cnt); + return (SYSCTL_OUT(req, &count, sizeof(count))); } SYSCTL_PROC(_vm, VM_TOTAL, vmtotal, CTLTYPE_OPAQUE|CTLFLAG_RD|CTLFLAG_MPSAFE, @@ -246,8 +254,8 @@ SYSCTL_NODE(_vm_stats, OID_AUTO, misc, C #define VM_STATS(parent, var, descr) \ SYSCTL_PROC(parent, OID_AUTO, var, \ - CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, vcnt, \ - "IU", descr) + CTLTYPE_UINT | CTLFLAG_RD | CTLFLAG_MPSAFE, &vm_cnt.var, 0, \ + cnt_sysctl, "IU", descr) #define VM_STATS_VM(var, descr) VM_STATS(_vm_stats_vm, var, descr) #define VM_STATS_SYS(var, descr) VM_STATS(_vm_stats_sys, var, descr) @@ -271,9 +279,10 @@ VM_STATS_VM(v_vnodeout, "Vnode pager pag VM_STATS_VM(v_vnodepgsin, "Vnode pages paged in"); VM_STATS_VM(v_vnodepgsout, "Vnode pages paged out"); VM_STATS_VM(v_intrans, "In transit page faults"); -VM_STATS_VM(v_reactivated, "Pages reactivated from free list"); +VM_STATS_VM(v_reactivated, "Pages reactivated by pagedaemon"); VM_STATS_VM(v_pdwakeups, "Pagedaemon wakeups"); VM_STATS_VM(v_pdpages, "Pages analyzed by pagedaemon"); +VM_STATS_VM(v_pdshortfalls, "Page reclamation shortfalls"); VM_STATS_VM(v_tcached, "Total pages cached"); VM_STATS_VM(v_dfree, "Pages freed by pagedaemon"); VM_STATS_VM(v_pfree, "Pages freed by exiting processes"); @@ -288,6 +297,7 @@ VM_STATS_VM(v_wire_count, "Wired pages") VM_STATS_VM(v_active_count, "Active pages"); VM_STATS_VM(v_inactive_target, "Desired inactive pages"); VM_STATS_VM(v_inactive_count, "Inactive pages"); +VM_STATS_VM(v_laundry_count, "Pages eligible for laundering"); VM_STATS_VM(v_cache_count, "Pages on cache queue"); VM_STATS_VM(v_pageout_free_min, "Min pages reserved for kernel"); VM_STATS_VM(v_interrupt_free_min, "Reserved pages for interrupt code"); Modified: stable/11/sys/vm/vm_mmap.c ============================================================================== --- stable/11/sys/vm/vm_mmap.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_mmap.c Tue May 23 07:27:30 2017 (r318716) @@ -849,9 +849,6 @@ RestartScan: pindex = OFF_TO_IDX(current->offset + (addr - current->start)); m = vm_page_lookup(object, pindex); - if (m == NULL && - vm_page_is_cached(object, pindex)) - mincoreinfo = MINCORE_INCORE; if (m != NULL && m->valid == 0) m = NULL; if (m != NULL) Modified: stable/11/sys/vm/vm_object.c ============================================================================== --- stable/11/sys/vm/vm_object.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_object.c Tue May 23 07:27:30 2017 (r318716) @@ -178,9 +178,6 @@ vm_object_zdtor(void *mem, int size, voi ("object %p has reservations", object)); #endif - KASSERT(vm_object_cache_is_empty(object), - ("object %p has cached pages", - object)); KASSERT(object->paging_in_progress == 0, ("object %p paging_in_progress = %d", object, object->paging_in_progress)); @@ -208,12 +205,9 @@ vm_object_zinit(void *mem, int size, int object->type = OBJT_DEAD; object->ref_count = 0; object->rtree.rt_root = 0; - object->rtree.rt_flags = 0; object->paging_in_progress = 0; object->resident_page_count = 0; object->shadow_count = 0; - object->cache.rt_root = 0; - object->cache.rt_flags = 0; mtx_lock(&vm_object_list_mtx); TAILQ_INSERT_TAIL(&vm_object_list, object, object_list); @@ -792,8 +786,6 @@ vm_object_terminate(vm_object_t object) if (__predict_false(!LIST_EMPTY(&object->rvq))) vm_reserv_break_all(object); #endif - if (__predict_false(!vm_object_cache_is_empty(object))) - vm_page_cache_free(object, 0, 0); KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT || object->type == OBJT_SWAP, @@ -1135,13 +1127,6 @@ shadowlookup: } else if ((tobject->flags & OBJ_UNMANAGED) != 0) goto unlock_tobject; m = vm_page_lookup(tobject, tpindex); - if (m == NULL && advise == MADV_WILLNEED) { - /* - * If the page is cached, reactivate it. - */ - m = vm_page_alloc(tobject, tpindex, VM_ALLOC_IFCACHED | - VM_ALLOC_NOBUSY); - } if (m == NULL) { /* * There may be swap even if there is no backing page @@ -1371,7 +1356,7 @@ retry: goto retry; } - /* vm_page_rename() will handle dirty and cache. */ + /* vm_page_rename() will dirty the page. */ if (vm_page_rename(m, new_object, idx)) { VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); @@ -1406,19 +1391,6 @@ retry: swap_pager_copy(orig_object, new_object, offidxstart, 0); TAILQ_FOREACH(m, &new_object->memq, listq) vm_page_xunbusy(m); - - /* - * Transfer any cached pages from orig_object to new_object. - * If swap_pager_copy() found swapped out pages within the - * specified range of orig_object, then it changed - * new_object's type to OBJT_SWAP when it transferred those - * pages to new_object. Otherwise, new_object's type - * should still be OBJT_DEFAULT and orig_object should not - * contain any cached pages within the specified range. - */ - if (__predict_false(!vm_object_cache_is_empty(orig_object))) - vm_page_cache_transfer(orig_object, offidxstart, - new_object); } VM_OBJECT_WUNLOCK(orig_object); VM_OBJECT_WUNLOCK(new_object); @@ -1471,6 +1443,13 @@ vm_object_scan_all_shadowed(vm_object_t backing_object = object->backing_object; + /* + * Initial conditions: + * + * We do not want to have to test for the existence of swap + * pages in the backing object. XXX but with the new swapper this + * would be pretty easy to do. + */ if (backing_object->type != OBJT_DEFAULT && backing_object->type != OBJT_SWAP) return (false); @@ -1622,8 +1601,7 @@ vm_object_collapse_scan(vm_object_t obje * backing object to the main object. * * If the page was mapped to a process, it can remain mapped - * through the rename. vm_page_rename() will handle dirty and - * cache. + * through the rename. vm_page_rename() will dirty the page. */ if (vm_page_rename(p, object, new_pindex)) { next = vm_object_collapse_scan_wait(object, NULL, next, @@ -1758,13 +1736,6 @@ vm_object_collapse(vm_object_t object) backing_object, object, OFF_TO_IDX(object->backing_object_offset), TRUE); - - /* - * Free any cached pages from backing_object. - */ - if (__predict_false( - !vm_object_cache_is_empty(backing_object))) - vm_page_cache_free(backing_object, 0, 0); } /* * Object now shadows whatever backing_object did. @@ -1893,7 +1864,7 @@ vm_object_page_remove(vm_object_t object (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED, ("vm_object_page_remove: illegal options for object %p", object)); if (object->resident_page_count == 0) - goto skipmemq; + return; vm_object_pip_add(object, 1); again: p = vm_page_find_least(object, start); @@ -1950,9 +1921,6 @@ next: vm_page_unlock(p); } vm_object_pip_wakeup(object); -skipmemq: - if (__predict_false(!vm_object_cache_is_empty(object))) - vm_page_cache_free(object, start, end); } /* @@ -2333,9 +2301,9 @@ sysctl_vm_object_list(SYSCTL_HANDLER_ARG * sysctl is only meant to give an * approximation of the system anyway. */ - if (m->queue == PQ_ACTIVE) + if (vm_page_active(m)) kvo.kvo_active++; - else if (m->queue == PQ_INACTIVE) + else if (vm_page_inactive(m)) kvo.kvo_inactive++; } Modified: stable/11/sys/vm/vm_object.h ============================================================================== --- stable/11/sys/vm/vm_object.h Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_object.h Tue May 23 07:27:30 2017 (r318716) @@ -79,17 +79,6 @@ * * vm_object_t Virtual memory object. * - * The root of cached pages pool is protected by both the per-object lock - * and the free pages queue mutex. - * On insert in the cache radix trie, the per-object lock is expected - * to be already held and the free pages queue mutex will be - * acquired during the operation too. - * On remove and lookup from the cache radix trie, only the free - * pages queue mutex is expected to be locked. - * These rules allow for reliably checking for the presence of cached - * pages with only the per-object lock held, thereby reducing contention - * for the free pages queue mutex. - * * List of locks * (c) const until freed * (o) per-object lock @@ -118,7 +107,6 @@ struct vm_object { vm_ooffset_t backing_object_offset;/* Offset in backing object */ TAILQ_ENTRY(vm_object) pager_object_list; /* list of all objects of this pager type */ LIST_HEAD(, vm_reserv) rvq; /* list of reservations */ - struct vm_radix cache; /* (o + f) root of the cache page radix trie */ void *handle; union { /* @@ -306,13 +294,6 @@ void vm_object_pip_wakeup(vm_object_t ob void vm_object_pip_wakeupn(vm_object_t object, short i); void vm_object_pip_wait(vm_object_t object, char *waitid); -static __inline boolean_t -vm_object_cache_is_empty(vm_object_t object) -{ - - return (vm_radix_is_empty(&object->cache)); -} - void umtx_shm_object_init(vm_object_t object); void umtx_shm_object_terminated(vm_object_t object); extern int umtx_shm_vnobj_persistent; Modified: stable/11/sys/vm/vm_page.c ============================================================================== --- stable/11/sys/vm/vm_page.c Tue May 23 07:26:45 2017 (r318715) +++ stable/11/sys/vm/vm_page.c Tue May 23 07:27:30 2017 (r318716) @@ -155,8 +155,7 @@ static int vm_pageout_pages_needed; static uma_zone_t fakepg_zone; -static struct vnode *vm_page_alloc_init(vm_page_t m); -static void vm_page_cache_turn_free(vm_page_t m); +static void vm_page_alloc_check(vm_page_t m); static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); static void vm_page_enqueue(uint8_t queue, vm_page_t m); static void vm_page_free_wakeup(void); @@ -391,6 +390,10 @@ vm_page_domain_init(struct vm_domain *vm "vm active pagequeue"; *__DECONST(u_int **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_vcnt) = &vm_cnt.v_active_count; + *__DECONST(char **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) = + "vm laundry pagequeue"; + *__DECONST(int **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_vcnt) = + &vm_cnt.v_laundry_count; vmd->vmd_page_count = 0; vmd->vmd_free_count = 0; vmd->vmd_segs = 0; @@ -1136,9 +1139,7 @@ void vm_page_dirty_KBI(vm_page_t m) { - /* These assertions refer to this operation by its public name. */ - KASSERT((m->flags & PG_CACHED) == 0, - ("vm_page_dirty: page in cache!")); + /* Refer to this operation by its public name. */ KASSERT(m->valid == VM_PAGE_BITS_ALL, ("vm_page_dirty: page is invalid!")); m->dirty = VM_PAGE_BITS_ALL; @@ -1262,9 +1263,8 @@ vm_page_insert_radixdone(vm_page_t m, vm /* * vm_page_remove: * - * Removes the given mem entry from the object/offset-page - * table and the object page list, but do not invalidate/terminate - * the backing store. + * Removes the specified page from its containing object, but does not + * invalidate any backing storage. * * The object must be locked. The page must be locked if it is managed. */ @@ -1272,6 +1272,7 @@ void vm_page_remove(vm_page_t m) { vm_object_t object; + vm_page_t mrem; if ((m->oflags & VPO_UNMANAGED) == 0) vm_page_assert_locked(m); @@ -1280,11 +1281,12 @@ vm_page_remove(vm_page_t m) VM_OBJECT_ASSERT_WLOCKED(object); if (vm_page_xbusied(m)) vm_page_xunbusy_maybelocked(m); + mrem = vm_radix_remove(&object->rtree, m->pindex); + KASSERT(mrem == m, ("removed page %p, expected page %p", mrem, m)); /* * Now remove from the object's list of backed pages. */ - vm_radix_remove(&object->rtree, m->pindex); TAILQ_REMOVE(&object->memq, m, listq); /* @@ -1433,9 +1435,7 @@ vm_page_replace(vm_page_t mnew, vm_objec * * Note: we *always* dirty the page. It is necessary both for the * fact that we moved it, and because we may be invalidating - * swap. If the page is on the cache, we have to deactivate it - * or vm_page_dirty() will panic. Dirty pages are not allowed - * on the cache. + * swap. * * The objects must be locked. */ @@ -1481,142 +1481,6 @@ vm_page_rename(vm_page_t m, vm_object_t } /* - * Convert all of the given object's cached pages that have a - * pindex within the given range into free pages. If the value - * zero is given for "end", then the range's upper bound is - * infinity. If the given object is backed by a vnode and it - * transitions from having one or more cached pages to none, the - * vnode's hold count is reduced. - */ -void -vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end) -{ - vm_page_t m; - boolean_t empty; - - mtx_lock(&vm_page_queue_free_mtx); - if (__predict_false(vm_radix_is_empty(&object->cache))) { - mtx_unlock(&vm_page_queue_free_mtx); - return; - } - while ((m = vm_radix_lookup_ge(&object->cache, start)) != NULL) { - if (end != 0 && m->pindex >= end) - break; - vm_radix_remove(&object->cache, m->pindex); - vm_page_cache_turn_free(m); - } - empty = vm_radix_is_empty(&object->cache); - mtx_unlock(&vm_page_queue_free_mtx); - if (object->type == OBJT_VNODE && empty) - vdrop(object->handle); -} - -/* - * Returns the cached page that is associated with the given - * object and offset. If, however, none exists, returns NULL. - * - * The free page queue must be locked. - */ -static inline vm_page_t -vm_page_cache_lookup(vm_object_t object, vm_pindex_t pindex) -{ - - mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - return (vm_radix_lookup(&object->cache, pindex)); -} - -/* - * Remove the given cached page from its containing object's - * collection of cached pages. - * - * The free page queue must be locked. - */ -static void -vm_page_cache_remove(vm_page_t m) -{ - - mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - KASSERT((m->flags & PG_CACHED) != 0, - ("vm_page_cache_remove: page %p is not cached", m)); - vm_radix_remove(&m->object->cache, m->pindex); - m->object = NULL; - vm_cnt.v_cache_count--; -} - -/* - * Transfer all of the cached pages with offset greater than or - * equal to 'offidxstart' from the original object's cache to the - * new object's cache. However, any cached pages with offset - * greater than or equal to the new object's size are kept in the - * original object. Initially, the new object's cache must be - * empty. Offset 'offidxstart' in the original object must - * correspond to offset zero in the new object. - * - * The new object must be locked. - */ -void -vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart, - vm_object_t new_object) -{ - vm_page_t m; - - /* - * Insertion into an object's collection of cached pages - * requires the object to be locked. In contrast, removal does - * not. - */ - VM_OBJECT_ASSERT_WLOCKED(new_object); - KASSERT(vm_radix_is_empty(&new_object->cache), - ("vm_page_cache_transfer: object %p has cached pages", - new_object)); - mtx_lock(&vm_page_queue_free_mtx); - while ((m = vm_radix_lookup_ge(&orig_object->cache, - offidxstart)) != NULL) { - /* - * Transfer all of the pages with offset greater than or - * equal to 'offidxstart' from the original object's - * cache to the new object's cache. - */ - if ((m->pindex - offidxstart) >= new_object->size) - break; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201705230727.v4N7RUYV078792>