Date: Wed, 28 Jun 2017 04:23:21 +0000 (UTC) From: Alan Cox <alc@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r320432 - in stable/10/sys: amd64/amd64 amd64/include i386/i386 i386/include Message-ID: <201706280423.v5S4NLQA097070@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alc Date: Wed Jun 28 04:23:20 2017 New Revision: 320432 URL: https://svnweb.freebsd.org/changeset/base/320432 Log: MFC r314310 Refine the fix from r312954. Specifically, add a new PDE-only flag, PG_PROMOTED, that indicates whether lingering 4KB page mappings might need to be flushed on a PDE change that restricts or destroys a 2MB page mapping. This flag allows the pmap to avoid range invalidations that are both unnecessary and costly. Modified: stable/10/sys/amd64/amd64/pmap.c stable/10/sys/amd64/include/pmap.h stable/10/sys/i386/i386/pmap.c stable/10/sys/i386/include/pmap.h Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/amd64/amd64/pmap.c ============================================================================== --- stable/10/sys/amd64/amd64/pmap.c Wed Jun 28 04:19:54 2017 (r320431) +++ stable/10/sys/amd64/amd64/pmap.c Wed Jun 28 04:23:20 2017 (r320432) @@ -455,6 +455,8 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); +static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, + pd_entry_t pde); static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va); static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask); @@ -1777,6 +1779,27 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_ } #endif /* !SMP */ +static void +pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde) +{ + + /* + * When the PDE has PG_PROMOTED set, the 2MB page mapping was created + * by a promotion that did not invalidate the 512 4KB page mappings + * that might exist in the TLB. Consequently, at this point, the TLB + * may hold both 4KB and 2MB page mappings for the address range [va, + * va + NBPDR). Therefore, the entire range must be invalidated here. + * In contrast, when PG_PROMOTED is clear, the TLB will not hold any + * 4KB page mappings for the address range [va, va + NBPDR), and so a + * single INVLPG suffices to invalidate the 2MB page mapping from the + * TLB. + */ + if ((pde & PG_PROMOTED) != 0) + pmap_invalidate_range(pmap, va, va + NBPDR - 1); + else + pmap_invalidate_page(pmap, va); +} + #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) void @@ -3418,7 +3441,8 @@ pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, v SLIST_INIT(&free); sva = trunc_2mpage(va); pmap_remove_pde(pmap, pde, sva, &free, lockp); - pmap_invalidate_range(pmap, sva, sva + NBPDR - 1); + if ((oldpde & PG_G) == 0) + pmap_invalidate_pde_page(pmap, sva, oldpde); pmap_free_zero_pages(&free); CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx" " in pmap %p", va, pmap); @@ -3559,25 +3583,8 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offse oldpde = pte_load_clear(pdq); if (oldpde & PG_W) pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; - - /* - * When workaround_erratum383 is false, a promotion to a 2M - * page mapping does not invalidate the 512 4K page mappings - * from the TLB. Consequently, at this point, the TLB may - * hold both 4K and 2M page mappings. Therefore, the entire - * range of addresses must be invalidated here. In contrast, - * when workaround_erratum383 is true, a promotion does - * invalidate the 512 4K page mappings, and so a single INVLPG - * suffices to invalidate the 2M page mapping. - */ - if ((oldpde & PG_G) != 0) { - if (workaround_erratum383) - pmap_invalidate_page(kernel_pmap, sva); - else - pmap_invalidate_range(kernel_pmap, sva, - sva + NBPDR - 1); - } - + if ((oldpde & PG_G) != 0) + pmap_invalidate_pde_page(kernel_pmap, sva, oldpde); pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); if (oldpde & PG_MANAGED) { CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); @@ -3930,16 +3937,16 @@ retry: if ((prot & VM_PROT_EXECUTE) == 0) newpde |= pg_nx; if (newpde != oldpde) { - if (!atomic_cmpset_long(pde, oldpde, newpde)) + /* + * As an optimization to future operations on this PDE, clear + * PG_PROMOTED. The impending invalidation will remove any + * lingering 4KB page mappings from the TLB. + */ + if (!atomic_cmpset_long(pde, oldpde, newpde & ~PG_PROMOTED)) goto retry; - if (oldpde & PG_G) { - /* See pmap_remove_pde() for explanation. */ - if (workaround_erratum383) - pmap_invalidate_page(kernel_pmap, sva); - else - pmap_invalidate_range(kernel_pmap, sva, - sva + NBPDR - 1); - } else + if ((oldpde & PG_G) != 0) + pmap_invalidate_pde_page(kernel_pmap, sva, oldpde); + else anychanged = TRUE; } return (anychanged); @@ -4210,7 +4217,7 @@ setpte: if (workaround_erratum383) pmap_update_pde(pmap, va, pde, PG_PS | newpde); else - pde_store(pde, PG_PS | newpde); + pde_store(pde, PG_PROMOTED | PG_PS | newpde); atomic_add_long(&pmap_pde_promotions, 1); CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx" @@ -4519,7 +4526,8 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); /* - * Map the superpage. + * Map the superpage. (This is not a promoted mapping; there will not + * be any lingering 4KB page mappings in the TLB.) */ pde_store(pde, newpde); Modified: stable/10/sys/amd64/include/pmap.h ============================================================================== --- stable/10/sys/amd64/include/pmap.h Wed Jun 28 04:19:54 2017 (r320431) +++ stable/10/sys/amd64/include/pmap.h Wed Jun 28 04:23:20 2017 (r320432) @@ -109,6 +109,7 @@ #define PG_MANAGED X86_PG_AVAIL2 #define EPT_PG_EMUL_V X86_PG_AVAIL(52) #define EPT_PG_EMUL_RW X86_PG_AVAIL(53) +#define PG_PROMOTED X86_PG_AVAIL(54) /* PDE only */ #define PG_FRAME (0x000ffffffffff000ul) #define PG_PS_FRAME (0x000fffffffe00000ul) Modified: stable/10/sys/i386/i386/pmap.c ============================================================================== --- stable/10/sys/i386/i386/pmap.c Wed Jun 28 04:19:54 2017 (r320431) +++ stable/10/sys/i386/i386/pmap.c Wed Jun 28 04:23:20 2017 (r320432) @@ -317,6 +317,8 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_page_t m, vm_prot_t prot, vm_page_t mpte); static void pmap_flush_page(vm_page_t m); static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); +static void pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, + pd_entry_t pde); static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static boolean_t pmap_is_modified_pvh(struct md_page *pvh); static boolean_t pmap_is_referenced_pvh(struct md_page *pvh); @@ -1215,6 +1217,27 @@ pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_ } #endif /* !SMP */ +static void +pmap_invalidate_pde_page(pmap_t pmap, vm_offset_t va, pd_entry_t pde) +{ + + /* + * When the PDE has PG_PROMOTED set, the 2- or 4MB page mapping was + * created by a promotion that did not invalidate the 512 or 1024 4KB + * page mappings that might exist in the TLB. Consequently, at this + * point, the TLB may hold both 4KB and 2- or 4MB page mappings for + * the address range [va, va + NBPDR). Therefore, the entire range + * must be invalidated here. In contrast, when PG_PROMOTED is clear, + * the TLB will not hold any 4KB page mappings for the address range + * [va, va + NBPDR), and so a single INVLPG suffices to invalidate the + * 2- or 4MB page mapping from the TLB. + */ + if ((pde & PG_PROMOTED) != 0) + pmap_invalidate_range(pmap, va, va + NBPDR - 1); + else + pmap_invalidate_page(pmap, va); +} + #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) void @@ -2724,7 +2747,8 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offse SLIST_INIT(&free); sva = trunc_4mpage(va); pmap_remove_pde(pmap, pde, sva, &free); - pmap_invalidate_range(pmap, sva, sva + NBPDR - 1); + if ((oldpde & PG_G) == 0) + pmap_invalidate_pde_page(pmap, sva, oldpde); pmap_free_zero_pages(&free); CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" " in pmap %p", va, pmap); @@ -2895,23 +2919,9 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offse /* * Machines that don't support invlpg, also don't support * PG_G. - * - * When workaround_erratum383 is false, a promotion to a 2M/4M - * page mapping does not invalidate the 512/1024 4K page mappings - * from the TLB. Consequently, at this point, the TLB may - * hold both 4K and 2M/4M page mappings. Therefore, the entire - * range of addresses must be invalidated here. In contrast, - * when workaround_erratum383 is true, a promotion does - * invalidate the 512/1024 4K page mappings, and so a single INVLPG - * suffices to invalidate the 2M/4M page mapping. */ - if ((oldpde & PG_G) != 0) { - if (workaround_erratum383) - pmap_invalidate_page(kernel_pmap, sva); - else - pmap_invalidate_range(kernel_pmap, sva, - sva + NBPDR - 1); - } + if ((oldpde & PG_G) != 0) + pmap_invalidate_pde_page(kernel_pmap, sva, oldpde); pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; if (oldpde & PG_MANAGED) { @@ -3220,16 +3230,16 @@ retry: newpde |= pg_nx; #endif if (newpde != oldpde) { - if (!pde_cmpset(pde, oldpde, newpde)) + /* + * As an optimization to future operations on this PDE, clear + * PG_PROMOTED. The impending invalidation will remove any + * lingering 4KB page mappings from the TLB. + */ + if (!pde_cmpset(pde, oldpde, newpde & ~PG_PROMOTED)) goto retry; - if (oldpde & PG_G) { - /* See pmap_remove_pde() for explanation. */ - if (workaround_erratum383) - pmap_invalidate_page(kernel_pmap, sva); - else - pmap_invalidate_range(kernel_pmap, sva, - sva + NBPDR - 1); - } else + if ((oldpde & PG_G) != 0) + pmap_invalidate_pde_page(kernel_pmap, sva, oldpde); + else anychanged = TRUE; } return (anychanged); @@ -3514,9 +3524,9 @@ setpte: if (workaround_erratum383) pmap_update_pde(pmap, va, pde, PG_PS | newpde); else if (pmap == kernel_pmap) - pmap_kenter_pde(va, PG_PS | newpde); + pmap_kenter_pde(va, PG_PROMOTED | PG_PS | newpde); else - pde_store(pde, PG_PS | newpde); + pde_store(pde, PG_PROMOTED | PG_PS | newpde); pmap_pde_promotions++; CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#x" @@ -3787,7 +3797,8 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE; /* - * Map the superpage. + * Map the superpage. (This is not a promoted mapping; there will not + * be any lingering 4KB page mappings in the TLB.) */ pde_store(pde, newpde); Modified: stable/10/sys/i386/include/pmap.h ============================================================================== --- stable/10/sys/i386/include/pmap.h Wed Jun 28 04:19:54 2017 (r320431) +++ stable/10/sys/i386/include/pmap.h Wed Jun 28 04:23:20 2017 (r320432) @@ -71,6 +71,7 @@ /* Our various interpretations of the above */ #define PG_W PG_AVAIL1 /* "Wired" pseudoflag */ #define PG_MANAGED PG_AVAIL2 +#define PG_PROMOTED PG_AVAIL3 /* PDE only */ #if defined(PAE) || defined(PAE_TABLES) #define PG_FRAME (0x000ffffffffff000ull) #define PG_PS_FRAME (0x000fffffffe00000ull)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201706280423.v5S4NLQA097070>