Date: Sat, 1 Jul 2017 09:38:52 +0000 (UTC) From: Andrew Turner <andrew@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r320529 - head/sys/arm64/arm64 Message-ID: <201707010938.v619cquC028333@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: andrew Date: Sat Jul 1 09:38:52 2017 New Revision: 320529 URL: https://svnweb.freebsd.org/changeset/base/320529 Log: Remove all calls to cpu_dcache_wb_range from the arm64 pmap code. These were unneeded as we tell the tlb the pagetables are in cached memory. This gives us a small, but statistically significant improvement over just removing the PTE_SYNC cases. While here remove PTE_SYNC, it's now unneeded. Sponsored by: DARPA, AFRL Modified: head/sys/arm64/arm64/pmap.c Modified: head/sys/arm64/arm64/pmap.c ============================================================================== --- head/sys/arm64/arm64/pmap.c Sat Jul 1 05:35:29 2017 (r320528) +++ head/sys/arm64/arm64/pmap.c Sat Jul 1 09:38:52 2017 (r320529) @@ -504,12 +504,6 @@ pmap_l3_valid(pt_entry_t l3) CTASSERT(L1_BLOCK == L2_BLOCK); -#if 0 -#define PTE_SYNC(pte) cpu_dcache_wb_range((vm_offset_t)pte, sizeof(*pte)) -#else -#define PTE_SYNC(pte) (void)0 -#endif - /* * Checks if the page is dirty. We currently lack proper tracking of this on * arm64 so for now assume is a page mapped as rw was accessed it is. @@ -594,8 +588,6 @@ pmap_bootstrap_dmap(vm_offset_t kern_l1, vm_paddr_t mi dmap_phys_max = pa; dmap_max_addr = va; - cpu_dcache_wb_range((vm_offset_t)pagetable_dmap, - PAGE_SIZE * DMAP_TABLES); cpu_tlb_flushID(); } @@ -624,11 +616,7 @@ pmap_bootstrap_l2(vm_offset_t l1pt, vm_offset_t va, vm /* Clean the L2 page table */ memset((void *)l2_start, 0, l2pt - l2_start); - cpu_dcache_wb_range(l2_start, l2pt - l2_start); - /* Flush the l1 table to ram */ - cpu_dcache_wb_range((vm_offset_t)l1, PAGE_SIZE); - return l2pt; } @@ -659,10 +647,7 @@ pmap_bootstrap_l3(vm_offset_t l1pt, vm_offset_t va, vm /* Clean the L2 page table */ memset((void *)l3_start, 0, l3pt - l3_start); - cpu_dcache_wb_range(l3_start, l3pt - l3_start); - cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); - return l3pt; } @@ -1131,7 +1116,6 @@ pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_ pte = pmap_l2_to_l3(pde, va); pmap_load_store(pte, (pa & ~L3_OFFSET) | attr); - PTE_SYNC(pte); va += PAGE_SIZE; pa += PAGE_SIZE; @@ -1161,7 +1145,6 @@ pmap_kremove(vm_offset_t va) KASSERT(lvl == 3, ("pmap_kremove: Invalid pte level %d", lvl)); pmap_load_clear(pte); - PTE_SYNC(pte); pmap_invalidate_page(kernel_pmap, va); } @@ -1184,7 +1167,6 @@ pmap_kremove_device(vm_offset_t sva, vm_size_t size) KASSERT(lvl == 3, ("Invalid device pagetable level: %d != 3", lvl)); pmap_load_clear(pte); - PTE_SYNC(pte); va += PAGE_SIZE; size -= PAGE_SIZE; @@ -1244,7 +1226,6 @@ pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) pa |= ATTR_XN; pte = pmap_l2_to_l3(pde, va); pmap_load_store(pte, pa); - PTE_SYNC(pte); va += L3_SIZE; } @@ -1271,7 +1252,6 @@ pmap_qremove(vm_offset_t sva, int count) ("Invalid device pagetable level: %d != 3", lvl)); if (pte != NULL) { pmap_load_clear(pte); - PTE_SYNC(pte); } va += PAGE_SIZE; @@ -1343,21 +1323,18 @@ _pmap_unwire_l3(pmap_t pmap, vm_offset_t va, vm_page_t l0 = pmap_l0(pmap, va); pmap_load_clear(l0); - PTE_SYNC(l0); } else if (m->pindex >= NUL2E) { /* l2 page */ pd_entry_t *l1; l1 = pmap_l1(pmap, va); pmap_load_clear(l1); - PTE_SYNC(l1); } else { /* l3 page */ pd_entry_t *l2; l2 = pmap_l2(pmap, va); pmap_load_clear(l2); - PTE_SYNC(l2); } pmap_resident_count_dec(pmap, 1); if (m->pindex < NUL2E) { @@ -1498,7 +1475,6 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str l0index = ptepindex - (NUL2E + NUL1E); l0 = &pmap->pm_l0[l0index]; pmap_load_store(l0, VM_PAGE_TO_PHYS(m) | L0_TABLE); - PTE_SYNC(l0); } else if (ptepindex >= NUL2E) { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1; @@ -1527,7 +1503,6 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str l1 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l0) & ~ATTR_MASK); l1 = &l1[ptepindex & Ln_ADDR_MASK]; pmap_load_store(l1, VM_PAGE_TO_PHYS(m) | L1_TABLE); - PTE_SYNC(l1); } else { vm_pindex_t l0index, l1index; pd_entry_t *l0, *l1, *l2; @@ -1574,7 +1549,6 @@ _pmap_alloc_l3(pmap_t pmap, vm_pindex_t ptepindex, str l2 = (pd_entry_t *)PHYS_TO_DMAP(pmap_load(l1) & ~ATTR_MASK); l2 = &l2[ptepindex & Ln_ADDR_MASK]; pmap_load_store(l2, VM_PAGE_TO_PHYS(m) | L2_TABLE); - PTE_SYNC(l2); } pmap_resident_count_inc(pmap, 1); @@ -1727,7 +1701,6 @@ pmap_growkernel(vm_offset_t addr) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); pmap_load_store(l1, paddr | L1_TABLE); - PTE_SYNC(l1); continue; /* try again */ } l2 = pmap_l1_to_l2(l1, kernel_vm_end); @@ -1749,7 +1722,6 @@ pmap_growkernel(vm_offset_t addr) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); pmap_load_store(l2, paddr | L2_TABLE); - PTE_SYNC(l2); pmap_invalidate_page(kernel_pmap, kernel_vm_end); kernel_vm_end = (kernel_vm_end + L2_SIZE) & ~L2_OFFSET; @@ -1883,7 +1855,6 @@ reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **l if ((tpte & ATTR_SW_WIRED) != 0) continue; tpte = pmap_load_clear(pte); - PTE_SYNC(pte); pmap_invalidate_page(pmap, va); m = PHYS_TO_VM_PAGE(tpte & ~ATTR_MASK); if (pmap_page_dirty(tpte)) @@ -2272,7 +2243,6 @@ pmap_remove_l3(pmap_t pmap, pt_entry_t *l3, vm_offset_ PMAP_LOCK_ASSERT(pmap, MA_OWNED); old_l3 = pmap_load_clear(l3); - PTE_SYNC(l3); pmap_invalidate_page(pmap, va); if (old_l3 & ATTR_SW_WIRED) pmap->pm_stats.wired_count -= 1; @@ -2493,7 +2463,6 @@ retry: pte = pmap_l2_to_l3(pde, pv->pv_va); tpte = pmap_load(pte); pmap_load_clear(pte); - PTE_SYNC(pte); pmap_invalidate_page(pmap, pv->pv_va); if (tpte & ATTR_SW_WIRED) pmap->pm_stats.wired_count--; @@ -2595,7 +2564,6 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t nbits |= ATTR_XN; pmap_set(l3p, nbits); - PTE_SYNC(l3p); /* XXX: Use pmap_invalidate_range */ pmap_invalidate_page(pmap, va); } @@ -2657,12 +2625,10 @@ pmap_update_entry(pmap_t pmap, pd_entry_t *pte, pd_ent /* Clear the old mapping */ pmap_load_clear(pte); - PTE_SYNC(pte); pmap_invalidate_range(pmap, va, va + size); /* Create the new mapping */ pmap_load_store(pte, newpte); - PTE_SYNC(pte); critical_exit(); intr_restore(intr); @@ -2886,7 +2852,6 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v l1_pa = VM_PAGE_TO_PHYS(l1_m); pmap_load_store(pde, l1_pa | L0_TABLE); - PTE_SYNC(pde); /* FALLTHROUGH */ case 0: /* Get the l1 pde to update */ @@ -2903,7 +2868,6 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v l2_pa = VM_PAGE_TO_PHYS(l2_m); pmap_load_store(pde, l2_pa | L1_TABLE); - PTE_SYNC(pde); /* FALLTHROUGH */ case 1: /* Get the l2 pde to update */ @@ -2919,7 +2883,6 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, v l3_pa = VM_PAGE_TO_PHYS(l3_m); pmap_load_store(pde, l3_pa | L2_TABLE); - PTE_SYNC(pde); break; } } @@ -3023,7 +2986,6 @@ validate: } } else { pmap_load_store(l3, new_l3); - PTE_SYNC(l3); pmap_invalidate_page(pmap, va); if (pmap_page_dirty(orig_l3) && (orig_l3 & ATTR_SW_MANAGED) != 0) @@ -3033,7 +2995,6 @@ validate: pmap_load_store(l3, new_l3); } - PTE_SYNC(l3); pmap_invalidate_page(pmap, va); if (pmap != pmap_kernel()) { @@ -3231,7 +3192,6 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, v if ((m->oflags & VPO_UNMANAGED) == 0) pa |= ATTR_SW_MANAGED; pmap_load_store(l3, pa); - PTE_SYNC(l3); pmap_invalidate_page(pmap, va); return (mpte); } @@ -3641,7 +3601,6 @@ pmap_remove_pages(pmap_t pmap) (uintmax_t)tpte)); pmap_load_clear(pte); - PTE_SYNC(pte); /* * Update the vm_page_t clean/reference bits. @@ -4416,7 +4375,6 @@ pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_ l2[i] = newl2 | phys; phys += L2_SIZE; } - cpu_dcache_wb_range((vm_offset_t)l2, PAGE_SIZE); KASSERT(l2[0] == ((oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK), ("Invalid l2 page (%lx != %lx)", l2[0], (oldl1 & ~ATTR_DESCR_MASK) | L2_BLOCK)); @@ -4494,7 +4452,6 @@ pmap_demote_l2_locked(pmap_t pmap, pt_entry_t *l2, vm_ l3[i] = newl3 | phys; phys += L3_SIZE; } - cpu_dcache_wb_range((vm_offset_t)l3, PAGE_SIZE); } KASSERT(l3[0] == ((oldl2 & ~ATTR_DESCR_MASK) | L3_PAGE), ("Invalid l3 page (%lx != %lx)", l3[0],
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201707010938.v619cquC028333>