Date: Fri, 4 Apr 2014 17:45:40 +0000 (UTC) From: Ian Lepore <ian@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r264129 - head/sys/arm/arm Message-ID: <201404041745.s34HjeUE010398@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: ian Date: Fri Apr 4 17:45:39 2014 New Revision: 264129 URL: http://svnweb.freebsd.org/changeset/base/264129 Log: Fix TLB maintenance issues for armv6 and armv7. - Add cpu_cpwait to comply with the convention. - Add missing TLB invalidations, especially in pmap_kenter & pmap_kremove with distinguishing between D and ID pages. - Modify pmap init/bootstrap invalidations to ID, just to be safe. - Fix TLB-inv and PTE_SYNC ordering. This combines changes submitted by ian@, cognet@, and Wojciech Macek, which have all been tested together as a unit. Modified: head/sys/arm/arm/pmap-v6.c Modified: head/sys/arm/arm/pmap-v6.c ============================================================================== --- head/sys/arm/arm/pmap-v6.c Fri Apr 4 17:39:05 2014 (r264128) +++ head/sys/arm/arm/pmap-v6.c Fri Apr 4 17:45:39 2014 (r264129) @@ -1047,6 +1047,7 @@ small_mappings: cpu_tlb_flushID_SE(pv->pv_va); else if (PTE_BEEN_REFD(opte)) cpu_tlb_flushD_SE(pv->pv_va); + cpu_cpwait(); } PMAP_UNLOCK(pmap); @@ -1134,8 +1135,8 @@ vector_page_setprot(int prot) *ptep |= L2_S_REF; pmap_set_prot(ptep, prot|VM_PROT_EXECUTE, 0); - - cpu_tlb_flushD_SE(vector_page); + PTE_SYNC(ptep); + cpu_tlb_flushID_SE(vector_page); cpu_cpwait(); } @@ -1643,8 +1644,8 @@ pmap_postinit(void) pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; *ptep = pte; PTE_SYNC(ptep); - cpu_tlb_flushD_SE(va); - + cpu_tlb_flushID_SE(va); + cpu_cpwait(); va += PAGE_SIZE; } pmap_init_l1(l1, pl1pt); @@ -1948,6 +1949,8 @@ pmap_bootstrap(vm_offset_t firstaddr, st pmap_init_l1(l1, kernel_l1pt); cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); + cpu_tlb_flushID(); + cpu_cpwait(); virtual_avail = round_page(virtual_avail); virtual_end = vm_max_kernel_address; @@ -2034,6 +2037,8 @@ pmap_grow_map(vm_offset_t va, pt_entry_t *ptep = L2_S_PROTO | pa | cache_mode | L2_S_REF; pmap_set_prot(ptep, VM_PROT_READ | VM_PROT_WRITE, 0); PTE_SYNC(ptep); + cpu_tlb_flushD_SE(va); + cpu_cpwait(); return (0); } @@ -2348,6 +2353,8 @@ pmap_kenter_section(vm_offset_t va, vm_o l1->l1_kva[L1_IDX(va)] = pd; PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); } + cpu_tlb_flushID_SE(va); + cpu_cpwait(); } /* @@ -2387,13 +2394,6 @@ pmap_kenter_internal(vm_offset_t va, vm_ ptep = &l2b->l2b_kva[l2pte_index(va)]; opte = *ptep; - if (l2pte_valid(opte)) { - cpu_tlb_flushD_SE(va); - cpu_cpwait(); - } else { - if (opte == 0) - l2b->l2b_occupancy++; - } if (flags & KENTER_CACHE) { *ptep = L2_S_PROTO | pa | pte_l2_s_cache_mode | L2_S_REF; @@ -2405,10 +2405,19 @@ pmap_kenter_internal(vm_offset_t va, vm_ 0); } + PTE_SYNC(ptep); + if (l2pte_valid(opte)) { + if (L2_S_EXECUTABLE(opte) || L2_S_EXECUTABLE(*ptep)) + cpu_tlb_flushID_SE(va); + else + cpu_tlb_flushD_SE(va); + } else { + if (opte == 0) + l2b->l2b_occupancy++; + } + PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", (uint32_t) ptep, opte, *ptep)); - PTE_SYNC(ptep); - cpu_cpwait(); } void @@ -2474,10 +2483,13 @@ pmap_kremove(vm_offset_t va) opte = *ptep; if (l2pte_valid(opte)) { va = va & ~PAGE_MASK; - cpu_tlb_flushD_SE(va); - cpu_cpwait(); *ptep = 0; PTE_SYNC(ptep); + if (L2_S_EXECUTABLE(opte)) + cpu_tlb_flushID_SE(va); + else + cpu_tlb_flushD_SE(va); + cpu_cpwait(); } } @@ -2710,6 +2722,7 @@ small_mappings: cpu_tlb_flushID(); else cpu_tlb_flushD(); + cpu_cpwait(); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); @@ -2763,6 +2776,7 @@ pmap_change_attr(vm_offset_t sva, vm_siz pmap_l2cache_wbinv_range(tmpva, pte & L2_S_FRAME, PAGE_SIZE); *ptep = pte; cpu_tlb_flushID_SE(tmpva); + cpu_cpwait(); dprintf("%s: for va:%x ptep:%x pte:%x\n", __func__, tmpva, (uint32_t)ptep, pte); @@ -2900,6 +2914,7 @@ pmap_protect(pmap_t pmap, vm_offset_t sv else if (is_refd) cpu_tlb_flushD(); + cpu_cpwait(); } rw_wunlock(&pvh_global_lock); @@ -3166,6 +3181,7 @@ validate: cpu_tlb_flushID_SE(va); else if (is_refd) cpu_tlb_flushD_SE(va); + cpu_cpwait(); } if ((pmap != pmap_kernel()) && (pmap == &curproc->p_vmspace->vm_pmap)) @@ -3713,6 +3729,7 @@ pmap_remove_section(pmap_t pmap, vm_offs cpu_tlb_flushID_SE(sva); else cpu_tlb_flushD_SE(sva); + cpu_cpwait(); } /* @@ -3885,6 +3902,7 @@ pmap_promote_section(pmap_t pmap, vm_off cpu_tlb_flushID(); else cpu_tlb_flushD(); + cpu_cpwait(); pmap_section_promotions++; CTR2(KTR_PMAP, "pmap_promote_section: success for va %#x" @@ -4009,6 +4027,7 @@ pmap_demote_section(pmap_t pmap, vm_offs cpu_tlb_flushID_SE(va); else if (L1_S_REFERENCED(l1pd)) cpu_tlb_flushD_SE(va); + cpu_cpwait(); pmap_section_demotions++; CTR2(KTR_PMAP, "pmap_demote_section: success for va %#x" @@ -4380,6 +4399,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva } } + *ptep = 0; + PTE_SYNC(ptep); if (pmap_is_current(pmap)) { total++; if (total < PMAP_REMOVE_CLEAN_LIST_SIZE) { @@ -4390,8 +4411,6 @@ pmap_remove(pmap_t pmap, vm_offset_t sva } else if (total == PMAP_REMOVE_CLEAN_LIST_SIZE) flushall = 1; } - *ptep = 0; - PTE_SYNC(ptep); sva += PAGE_SIZE; ptep++; @@ -4404,6 +4423,8 @@ pmap_remove(pmap_t pmap, vm_offset_t sva rw_wunlock(&pvh_global_lock); if (flushall) cpu_tlb_flushID(); + cpu_cpwait(); + PMAP_UNLOCK(pmap); } @@ -4920,6 +4941,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva cpu_tlb_flushID_SE(sva); else if (PTE_BEEN_REFD(opte)) cpu_tlb_flushD_SE(sva); + cpu_cpwait(); } } }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201404041745.s34HjeUE010398>