Date: Tue, 15 Dec 2015 15:22:33 +0000 (UTC) From: Svatopluk Kraus <skra@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r292269 - head/sys/arm/arm Message-ID: <201512151522.tBFFMXfP077405@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: skra Date: Tue Dec 15 15:22:33 2015 New Revision: 292269 URL: https://svnweb.freebsd.org/changeset/base/292269 Log: Replace all postponed TLB flushes by immediate ones except the one in pmap_remove_pages(). Some points were considered: (1) There is no range TLB flush cp15 function. (2) There is no target selection for hardware TLB flush broadcasting. (3) Some memory ranges could be mapped sparsely. (4) Some memory ranges could be quite large. Tested by buildworld on RPi2 and Jetson TK1, i.e. 4 core platforms. It turned out that the buildworld time is faster. On the other hand, when the postponed TLB flush was also removed from pmap_remove_pages(), the result was worse. But pmap_remove_pages() is called for removing all user mapping from a process, thus it's quite expected. Note that the postponed TLB flushes came here from i386 pmap where hardware TLB flush broadcasting is not available. Approved by: kib (mentor) Modified: head/sys/arm/arm/pmap-v6-new.c Modified: head/sys/arm/arm/pmap-v6-new.c ============================================================================== --- head/sys/arm/arm/pmap-v6-new.c Tue Dec 15 15:19:06 2015 (r292268) +++ head/sys/arm/arm/pmap-v6-new.c Tue Dec 15 15:22:33 2015 (r292269) @@ -2748,7 +2748,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) TAILQ_REMOVE(&pv_chunks, pc, pc_lru); if (pmap != pc->pc_pmap) { if (pmap != NULL) { - pmap_tlb_flush_ng(pmap); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } @@ -2786,8 +2785,7 @@ pmap_pv_reclaim(pmap_t locked_pmap) KASSERT(tpte2 != 0, ("pmap_pv_reclaim: pmap %p va %#x zero pte", pmap, va)); - if (pte2_is_global(tpte2)) - tlb_flush(va); + pmap_tlb_flush(pmap, va); m = PHYS_TO_VM_PAGE(pte2_pa(tpte2)); if (pte2_is_dirty(tpte2)) vm_page_dirty(m); @@ -2845,7 +2843,6 @@ pmap_pv_reclaim(pmap_t locked_pmap) out: TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); if (pmap != NULL) { - pmap_tlb_flush_ng(pmap); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } @@ -3375,18 +3372,16 @@ pmap_remove_pte1(pmap_t pmap, pt1_entry_ KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); + /* + * Clear and invalidate the mapping. It should occupy one and only TLB + * entry. So, pmap_tlb_flush() called with aligned address should be + * sufficient. + */ opte1 = pte1_load_clear(pte1p); + pmap_tlb_flush(pmap, sva); + if (pte1_is_wired(opte1)) pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE; - - /* - * If the mapping was global, invalidate it even if given pmap - * is not active (kernel_pmap is active always). The mapping should - * occupy one and only TLB entry. So, pmap_tlb_flush() called - * with aligned address should be sufficient. - */ - if (pte1_is_global(opte1)) - tlb_flush(sva); pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE; if (pte1_is_managed(opte1)) { pvh = pa_to_pvh(pte1_pa(opte1)); @@ -3470,7 +3465,6 @@ pmap_demote_pte1(pmap_t pmap, pt1_entry_ VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { SLIST_INIT(&free); pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free); - pmap_tlb_flush(pmap, pte1_trunc(va)); pmap_free_zero_pages(&free); CTR3(KTR_PMAP, "%s: failure for va %#x in pmap %p", __func__, va, pmap); @@ -3856,17 +3850,15 @@ pmap_remove_pte2(pmap_t pmap, pt2_entry_ rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); + /* Clear and invalidate the mapping. */ opte2 = pte2_load_clear(pte2p); + pmap_tlb_flush(pmap, va); + KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %#x not link pte2 %#x", __func__, pmap, va, opte2)); + if (opte2 & PTE2_W) pmap->pm_stats.wired_count -= 1; - /* - * If the mapping was global, invalidate it even if given pmap - * is not active (kernel_pmap is active always). - */ - if (pte2_is_global(opte2)) - tlb_flush(va); pmap->pm_stats.resident_count -= 1; if (pte2_is_managed(opte2)) { m = PHYS_TO_VM_PAGE(pte2_pa(opte2)); @@ -3895,7 +3887,6 @@ pmap_remove_page(pmap_t pmap, vm_offset_ !pte2_is_valid(pte2_load(pte2p))) return; pmap_remove_pte2(pmap, pte2p, va, free); - pmap_tlb_flush(pmap, va); } /* @@ -3911,7 +3902,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; struct spglist free; - int anyvalid; /* * Perform an unsynchronized read. This is, however, safe. @@ -3919,7 +3909,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva if (pmap->pm_stats.resident_count == 0) return; - anyvalid = 0; SLIST_INIT(&free); rw_wlock(&pvh_global_lock); @@ -3964,12 +3953,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { - /* - * The TLB entry for global mapping is - * invalidated by pmap_remove_pte1(). - */ - if (!pte1_is_global(pte1)) - anyvalid = 1; pmap_remove_pte1(pmap, pte1p, sva, &free); continue; } else if (!pmap_demote_pte1(pmap, pte1p, sva)) { @@ -4000,21 +3983,12 @@ pmap_remove(pmap_t pmap, vm_offset_t sva pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) continue; - - /* - * The TLB entry for global mapping is invalidated - * by pmap_remove_pte2(). - */ - if (!pte2_is_global(pte2)) - anyvalid = 1; if (pmap_remove_pte2(pmap, pte2p, sva, &free)) break; } } out: sched_unpin(); - if (anyvalid) - pmap_tlb_flush_ng(pmap); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); @@ -4070,6 +4044,7 @@ small_mappings: "a 1mpage in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); opte2 = pte2_load_clear(pte2p); + pmap_tlb_flush(pmap, pv->pv_va); KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %x zero pte2", __func__, pmap, pv->pv_va)); if (pte2_is_wired(opte2)) @@ -4083,7 +4058,6 @@ small_mappings: if (pte2_is_dirty(opte2)) vm_page_dirty(m); pmap_unuse_pt2(pmap, pv->pv_va, &free); - pmap_tlb_flush(pmap, pv->pv_va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); @@ -4602,19 +4576,17 @@ pmap_object_init_pt(pmap_t pmap, vm_offs /* * Do the things to protect a 1mpage in a process. */ -static boolean_t +static void pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, vm_prot_t prot) { pt1_entry_t npte1, opte1; vm_offset_t eva, va; vm_page_t m; - boolean_t anychanged; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); - anychanged = FALSE; retry: opte1 = npte1 = pte1_load(pte1p); if (pte1_is_managed(opte1)) { @@ -4638,12 +4610,8 @@ retry: if (npte1 != opte1) { if (!pte1_cmpset(pte1p, opte1, npte1)) goto retry; - if (pte1_is_global(opte1)) - tlb_flush(sva); - else - anychanged = TRUE; + pmap_tlb_flush(pmap, sva); } - return (anychanged); } /* @@ -4653,7 +4621,7 @@ retry: void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { - boolean_t anychanged, pv_lists_locked; + boolean_t pv_lists_locked; vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, opte2, npte2; @@ -4676,7 +4644,6 @@ resume: rw_wlock(&pvh_global_lock); sched_pin(); } - anychanged = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = nextva) { @@ -4703,19 +4670,12 @@ resume: * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { - /* - * The TLB entry for global mapping is - * invalidated by pmap_protect_pte1(). - */ - if (pmap_protect_pte1(pmap, pte1p, sva, prot)) - anychanged = TRUE; + pmap_protect_pte1(pmap, pte1p, sva, prot); continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { - if (anychanged) - pmap_tlb_flush_ng(pmap); PMAP_UNLOCK(pmap); goto resume; } @@ -4778,16 +4738,10 @@ retry: if (!pte2_cmpset(pte2p, opte2, npte2)) goto retry; - - if (pte2_is_global(opte2)) - tlb_flush(sva); - else - anychanged = TRUE; + pmap_tlb_flush(pmap, sva); } } } - if (anychanged) - pmap_tlb_flush_ng(pmap); if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); @@ -5294,7 +5248,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva pt2_entry_t *pte2p, pte2; vm_offset_t pdnxt; vm_page_t m; - boolean_t anychanged, pv_lists_locked; + boolean_t pv_lists_locked; if (advice != MADV_DONTNEED && advice != MADV_FREE) return; @@ -5306,7 +5260,6 @@ resume: rw_wlock(&pvh_global_lock); sched_pin(); } - anychanged = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = pdnxt) { pdnxt = pte1_trunc(sva + PTE1_SIZE); @@ -5322,8 +5275,6 @@ resume: if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { - if (anychanged) - pmap_tlb_flush_ng(pmap); PMAP_UNLOCK(pmap); goto resume; } @@ -5348,7 +5299,6 @@ resume: KASSERT(pte2_is_valid(pte2_load(pte2p)), ("%s: invalid PTE2", __func__)); pmap_remove_pte2(pmap, pte2p, sva, NULL); - anychanged = TRUE; } } if (pdnxt > eva) @@ -5374,14 +5324,9 @@ resume: pte2_clear_bit(pte2p, PTE2_A); else continue; - if (pte2_is_global(pte2)) - tlb_flush(sva); - else - anychanged = TRUE; + pmap_tlb_flush(pmap, sva); } } - if (anychanged) - pmap_tlb_flush_ng(pmap); if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201512151522.tBFFMXfP077405>