From owner-svn-src-head@FreeBSD.ORG Thu Jun 18 20:42:38 2009 Return-Path: Delivered-To: svn-src-head@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 474C51065673; Thu, 18 Jun 2009 20:42:38 +0000 (UTC) (envelope-from thompsa@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 33E3F8FC1F; Thu, 18 Jun 2009 20:42:38 +0000 (UTC) (envelope-from thompsa@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n5IKgcE6022370; Thu, 18 Jun 2009 20:42:38 GMT (envelope-from thompsa@svn.freebsd.org) Received: (from thompsa@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n5IKgc3t022366; Thu, 18 Jun 2009 20:42:38 GMT (envelope-from thompsa@svn.freebsd.org) Message-Id: <200906182042.n5IKgc3t022366@svn.freebsd.org> From: Andrew Thompson Date: Thu, 18 Jun 2009 20:42:38 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r194459 - in head/sys: arm/arm arm/include vm X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 18 Jun 2009 20:42:38 -0000 Author: thompsa Date: Thu Jun 18 20:42:37 2009 New Revision: 194459 URL: http://svn.freebsd.org/changeset/base/194459 Log: Track the kernel mapping of a physical page by a new entry in vm_page structure. When the page is shared, the kernel mapping becomes a special type of managed page to force the cache off the page mappings. This is needed to avoid stale entries on all ARM VIVT caches, and VIPT caches with cache color issue. Submitted by: Mark Tinguely Reviewed by: alc Tested by: Grzegorz Bernacki, thompsa Modified: head/sys/arm/arm/pmap.c head/sys/arm/include/pmap.h head/sys/vm/vm_phys.c Modified: head/sys/arm/arm/pmap.c ============================================================================== --- head/sys/arm/arm/pmap.c Thu Jun 18 19:58:05 2009 (r194458) +++ head/sys/arm/arm/pmap.c Thu Jun 18 20:42:37 2009 (r194459) @@ -200,8 +200,7 @@ static pv_entry_t pmap_get_pv_entry(void static void pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t, int); -static __inline void pmap_fix_cache(struct vm_page *, pmap_t, - vm_offset_t); +static void pmap_fix_cache(struct vm_page *, pmap_t, vm_offset_t); static void pmap_alloc_l1(pmap_t); static void pmap_free_l1(pmap_t); static void pmap_use_l1(pmap_t); @@ -406,7 +405,7 @@ int pmap_needs_pte_sync; #define pmap_is_current(pm) ((pm) == pmap_kernel() || \ curproc->p_vmspace->vm_map.pmap == (pm)) -static uma_zone_t pvzone; +static uma_zone_t pvzone = NULL; uma_zone_t l2zone; static uma_zone_t l2table_zone; static vm_offset_t pmap_kernel_l2dtable_kva; @@ -1451,6 +1450,7 @@ pmap_fix_cache(struct vm_page *pg, pmap_ * kernel writable or kernel readable with writable user entry */ if ((kwritable && entries) || + (kwritable > 1) || ((kwritable != writable) && kentries && (pv->pv_pmap == pmap_kernel() || (pv->pv_flags & PVF_WRITE) || @@ -1472,7 +1472,8 @@ pmap_fix_cache(struct vm_page *pg, pmap_ continue; } /* user is no longer sharable and writable */ - if (pm != pmap_kernel() && (pv->pv_pmap == pm) && + if (pm != pmap_kernel() && + (pv->pv_pmap == pm || pv->pv_pmap == pmap_kernel()) && !pmwc && (pv->pv_flags & PVF_NC)) { pv->pv_flags &= ~(PVF_NC | PVF_MWC); @@ -1674,7 +1675,29 @@ pmap_enter_pv(struct vm_page *pg, struct vm_offset_t va, u_int flags) { + int km; + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + + if (pg->md.pv_kva) { + /* PMAP_ASSERT_LOCKED(pmap_kernel()); */ + pve->pv_pmap = pmap_kernel(); + pve->pv_va = pg->md.pv_kva; + pve->pv_flags = PVF_WRITE | PVF_UNMAN; + pg->md.pv_kva = 0; + + TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); + TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); + if ((km = PMAP_OWNED(pmap_kernel()))) + PMAP_UNLOCK(pmap_kernel()); + vm_page_unlock_queues(); + if ((pve = pmap_get_pv_entry()) == NULL) + panic("pmap_kenter_internal: no pv entries"); + vm_page_lock_queues(); + if (km) + PMAP_LOCK(pmap_kernel()); + } + PMAP_ASSERT_LOCKED(pm); pve->pv_pmap = pm; pve->pv_va = va; @@ -1742,6 +1765,7 @@ static void pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) { + struct pv_entry *pv; mtx_assert(&vm_page_queue_mtx, MA_OWNED); PMAP_ASSERT_LOCKED(pm); TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); @@ -1766,6 +1790,20 @@ pmap_nuke_pv(struct vm_page *pg, pmap_t vm_page_flag_clear(pg, PG_WRITEABLE); } } + pv = TAILQ_FIRST(&pg->md.pv_list); + if (pv != NULL && (pv->pv_flags & PVF_UNMAN) && + TAILQ_NEXT(pv, pv_list) == NULL) { + pg->md.pv_kva = pv->pv_va; + /* a recursive pmap_nuke_pv */ + TAILQ_REMOVE(&pg->md.pv_list, pv, pv_list); + TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist); + if (pv->pv_flags & PVF_WIRED) + --pm->pm_stats.wired_count; + pg->md.pvh_attrs &= ~PVF_REF; + pg->md.pvh_attrs &= ~PVF_MOD; + vm_page_flag_clear(pg, PG_WRITEABLE); + pmap_free_pv_entry(pv); + } } static struct pv_entry * @@ -1784,6 +1822,9 @@ pmap_remove_pv(struct vm_page *pg, pmap_ pve = TAILQ_NEXT(pve, pv_list); } + if (pve == NULL && pg->md.pv_kva == va) + pg->md.pv_kva = 0; + return(pve); /* return removed pve */ } /* @@ -2711,8 +2752,8 @@ pmap_remove_pages(pmap_t pmap) cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { - if (pv->pv_flags & PVF_WIRED) { - /* The page is wired, cannot remove it now. */ + if (pv->pv_flags & PVF_WIRED || pv->pv_flags & PVF_UNMAN) { + /* Cannot remove wired or unmanaged pages now. */ npv = TAILQ_NEXT(pv, pv_plist); continue; } @@ -2822,6 +2863,9 @@ pmap_kenter_internal(vm_offset_t va, vm_ struct l2_bucket *l2b; pt_entry_t *pte; pt_entry_t opte; + struct pv_entry *pve; + vm_page_t m; + PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", (uint32_t) va, (uint32_t) pa)); @@ -2835,10 +2879,7 @@ pmap_kenter_internal(vm_offset_t va, vm_ PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", (uint32_t) pte, opte, *pte)); if (l2pte_valid(opte)) { - cpu_dcache_wbinv_range(va, PAGE_SIZE); - cpu_l2cache_wbinv_range(va, PAGE_SIZE); - cpu_tlb_flushD_SE(va); - cpu_cpwait(); + pmap_kremove(va); } else { if (opte == 0) l2b->l2b_occupancy++; @@ -2850,6 +2891,33 @@ pmap_kenter_internal(vm_offset_t va, vm_ if (flags & KENTER_USER) *pte |= L2_S_PROT_U; PTE_SYNC(pte); + + /* kernel direct mappings can be shared, so use a pv_entry + * to ensure proper caching. + * + * The pvzone is used to delay the recording of kernel + * mappings until the VM is running. + * + * This expects the physical memory to have vm_page_array entry. + */ + if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa))) { + vm_page_lock_queues(); + if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva) { + /* release vm_page lock for pv_entry UMA */ + vm_page_unlock_queues(); + if ((pve = pmap_get_pv_entry()) == NULL) + panic("pmap_kenter_internal: no pv entries"); + vm_page_lock_queues(); + PMAP_LOCK(pmap_kernel()); + pmap_enter_pv(m, pve, pmap_kernel(), va, + PVF_WRITE | PVF_UNMAN); + pmap_fix_cache(m, pmap_kernel(), va); + PMAP_UNLOCK(pmap_kernel()); + } else { + m->md.pv_kva = va; + } + vm_page_unlock_queues(); + } } void @@ -2886,6 +2954,9 @@ pmap_kremove(vm_offset_t va) { struct l2_bucket *l2b; pt_entry_t *pte, opte; + struct pv_entry *pve; + vm_page_t m; + vm_offset_t pa; l2b = pmap_get_l2_bucket(pmap_kernel(), va); if (!l2b) @@ -2894,6 +2965,25 @@ pmap_kremove(vm_offset_t va) pte = &l2b->l2b_kva[l2pte_index(va)]; opte = *pte; if (l2pte_valid(opte)) { + /* pa = vtophs(va) taken from pmap_extract() */ + switch (opte & L2_TYPE_MASK) { + case L2_TYPE_L: + pa = (opte & L2_L_FRAME) | (va & L2_L_OFFSET); + break; + default: + pa = (opte & L2_S_FRAME) | (va & L2_S_OFFSET); + break; + } + /* note: should never have to remove an allocation + * before the pvzone is initialized. + */ + vm_page_lock_queues(); + PMAP_LOCK(pmap_kernel()); + if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) && + (pve = pmap_remove_pv(m, pmap_kernel(), va))) + pmap_free_pv_entry(pve); + PMAP_UNLOCK(pmap_kernel()); + vm_page_unlock_queues(); cpu_dcache_wbinv_range(va, PAGE_SIZE); cpu_l2cache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); @@ -3137,16 +3227,24 @@ pmap_remove_all(vm_page_t m) cpu_l2cache_inv_range(pv->pv_va, PAGE_SIZE); } - l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); - KASSERT(l2b != NULL, ("No l2 bucket")); - ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; - *ptep = 0; - PTE_SYNC_CURRENT(pv->pv_pmap, ptep); - pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); - if (pv->pv_flags & PVF_WIRED) - pv->pv_pmap->pm_stats.wired_count--; - pv->pv_pmap->pm_stats.resident_count--; - flags |= pv->pv_flags; + if (pv->pv_flags & PVF_UNMAN) { + /* remove the pv entry, but do not remove the mapping + * and remember this is a kernel mapped page + */ + m->md.pv_kva = pv->pv_va; + } else { + /* remove the mapping and pv entry */ + l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); + KASSERT(l2b != NULL, ("No l2 bucket")); + ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; + *ptep = 0; + PTE_SYNC_CURRENT(pv->pv_pmap, ptep); + pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); + if (pv->pv_flags & PVF_WIRED) + pv->pv_pmap->pm_stats.wired_count--; + pv->pv_pmap->pm_stats.resident_count--; + flags |= pv->pv_flags; + } pmap_nuke_pv(m, pv->pv_pmap, pv); PMAP_UNLOCK(pv->pv_pmap); pmap_free_pv_entry(pv); @@ -3428,25 +3526,19 @@ do_l2b_alloc: * It is part of our managed memory so we * must remove it from the PV list */ - pve = pmap_remove_pv(opg, pmap, va); - if (m && (m->flags & (PG_UNMANAGED | PG_FICTITIOUS)) && - pve) - pmap_free_pv_entry(pve); - else if (!pve && - !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) - pve = pmap_get_pv_entry(); - KASSERT(pve != NULL || m->flags & (PG_UNMANAGED | - PG_FICTITIOUS), ("No pv")); - oflags = pve->pv_flags; - + if ((pve = pmap_remove_pv(opg, pmap, va))) { + + /* note for patch: the oflags/invalidation was moved + * because PG_FICTITIOUS pages could free the pve + */ + oflags = pve->pv_flags; /* * If the old mapping was valid (ref/mod * emulation creates 'invalid' mappings * initially) then make sure to frob * the cache. */ - if ((oflags & PVF_NC) == 0 && - l2pte_valid(opte)) { + if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) { if (PV_BEEN_EXECD(oflags)) { pmap_idcache_wbinv_range(pmap, va, PAGE_SIZE); @@ -3456,15 +3548,43 @@ do_l2b_alloc: PAGE_SIZE, TRUE, (oflags & PVF_WRITE) == 0); } - } - } else if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) - if ((pve = pmap_get_pv_entry()) == NULL) { - panic("pmap_enter: no pv entries"); - } - if (m && !(m->flags & (PG_UNMANAGED | PG_FICTITIOUS))) { + } + + /* free/allocate a pv_entry for UNMANAGED pages if + * this physical page is not/is already mapped. + */ + + if (m && ((m->flags & PG_FICTITIOUS) || + ((m->flags & PG_UNMANAGED) && + !m->md.pv_kva && + TAILQ_EMPTY(&m->md.pv_list)))) { + pmap_free_pv_entry(pve); + pve = NULL; + } + } else if (m && !(m->flags & PG_FICTITIOUS) && + (!(m->flags & PG_UNMANAGED) || m->md.pv_kva || + !TAILQ_EMPTY(&m->md.pv_list))) + pve = pmap_get_pv_entry(); + } else if (m && !(m->flags & PG_FICTITIOUS) && + (!(m->flags & PG_UNMANAGED) || m->md.pv_kva || + !TAILQ_EMPTY(&m->md.pv_list))) + pve = pmap_get_pv_entry(); + + if (m && !(m->flags & PG_FICTITIOUS)) { KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, - ("pmap_enter: managed mapping within the clean submap")); - pmap_enter_pv(m, pve, pmap, va, nflags); + ("pmap_enter: managed mapping within the clean submap")); + if (m->flags & PG_UNMANAGED) { + if (!TAILQ_EMPTY(&m->md.pv_list) || + m->md.pv_kva) { + KASSERT(pve != NULL, ("No pv")); + nflags |= PVF_UNMAN; + pmap_enter_pv(m, pve, pmap, va, nflags); + } else + m->md.pv_kva = va; + } else { + KASSERT(pve != NULL, ("No pv")); + pmap_enter_pv(m, pve, pmap, va, nflags); + } } } /* Modified: head/sys/arm/include/pmap.h ============================================================================== --- head/sys/arm/include/pmap.h Thu Jun 18 19:58:05 2009 (r194458) +++ head/sys/arm/include/pmap.h Thu Jun 18 20:42:37 2009 (r194459) @@ -94,6 +94,7 @@ struct pv_entry; struct md_page { int pvh_attrs; + vm_offset_t pv_kva; /* first kernel VA mapping */ TAILQ_HEAD(,pv_entry) pv_list; }; @@ -494,6 +495,7 @@ void pmap_use_minicache(vm_offset_t, vm_ #define PVF_EXEC 0x10 /* mapping is executable */ #define PVF_NC 0x20 /* mapping is non-cacheable */ #define PVF_MWC 0x40 /* mapping is used multiple times in userland */ +#define PVF_UNMAN 0x80 /* mapping is unmanaged */ void vector_page_setprot(int); Modified: head/sys/vm/vm_phys.c ============================================================================== --- head/sys/vm/vm_phys.c Thu Jun 18 19:58:05 2009 (r194458) +++ head/sys/vm/vm_phys.c Thu Jun 18 20:42:37 2009 (r194459) @@ -382,8 +382,7 @@ vm_phys_paddr_to_vm_page(vm_paddr_t pa) if (pa >= seg->start && pa < seg->end) return (&seg->first_page[atop(pa - seg->start)]); } - panic("vm_phys_paddr_to_vm_page: paddr %#jx is not in any segment", - (uintmax_t)pa); + return (NULL); } /*