Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 20 Apr 2012 00:45:24 +0000 (UTC)
From:      Nathan Whitehorn <nwhitehorn@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-9@freebsd.org
Subject:   svn commit: r234476 - in stable/9/sys: powerpc/aim vm
Message-ID:  <201204200045.q3K0jO65014820@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: nwhitehorn
Date: Fri Apr 20 00:45:23 2012
New Revision: 234476
URL: http://svn.freebsd.org/changeset/base/234476

Log:
  MFC r233949,233957,233964,234149,234155:
  More PMAP concurrency and performance improvements by only flushing icaches
  when needed and executing ptesync only when architecturally required.

Modified:
  stable/9/sys/powerpc/aim/machdep.c
  stable/9/sys/powerpc/aim/mmu_oea.c
  stable/9/sys/powerpc/aim/mmu_oea64.c
  stable/9/sys/powerpc/aim/moea64_native.c
  stable/9/sys/vm/vm_page.h
Directory Properties:
  stable/9/sys/   (props changed)

Modified: stable/9/sys/powerpc/aim/machdep.c
==============================================================================
--- stable/9/sys/powerpc/aim/machdep.c	Fri Apr 20 00:34:15 2012	(r234475)
+++ stable/9/sys/powerpc/aim/machdep.c	Fri Apr 20 00:45:23 2012	(r234476)
@@ -400,6 +400,9 @@ powerpc_init(vm_offset_t startkernel, vm
 		cacheline_size = 32;
 	}
 
+	/* Make sure the kernel icache is valid before we go too much further */
+	__syncicache((caddr_t)startkernel, endkernel - startkernel);
+
 	#ifndef __powerpc64__
 	/*
 	 * Figure out whether we need to use the 64 bit PMAP. This works by

Modified: stable/9/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- stable/9/sys/powerpc/aim/mmu_oea.c	Fri Apr 20 00:34:15 2012	(r234475)
+++ stable/9/sys/powerpc/aim/mmu_oea.c	Fri Apr 20 00:45:23 2012	(r234476)
@@ -1055,7 +1055,7 @@ moea_enter_locked(pmap_t pmap, vm_offset
 	struct		pvo_head *pvo_head;
 	uma_zone_t	zone;
 	vm_page_t	pg;
-	u_int		pte_lo, pvo_flags, was_exec;
+	u_int		pte_lo, pvo_flags;
 	int		error;
 
 	if (!moea_initialized) {
@@ -1063,13 +1063,11 @@ moea_enter_locked(pmap_t pmap, vm_offset
 		zone = moea_upvo_zone;
 		pvo_flags = 0;
 		pg = NULL;
-		was_exec = PTE_EXEC;
 	} else {
 		pvo_head = vm_page_to_pvoh(m);
 		pg = m;
 		zone = moea_mpvo_zone;
 		pvo_flags = PVO_MANAGED;
-		was_exec = 0;
 	}
 	if (pmap_bootstrapped)
 		mtx_assert(&vm_page_queue_mtx, MA_OWNED);
@@ -1085,18 +1083,6 @@ moea_enter_locked(pmap_t pmap, vm_offset
 		zone = moea_upvo_zone;
 	}
 
-	/*
-	 * If this is a managed page, and it's the first reference to the page,
-	 * clear the execness of the page.  Otherwise fetch the execness.
-	 */
-	if ((pg != NULL) && ((m->oflags & VPO_UNMANAGED) == 0)) {
-		if (LIST_EMPTY(pvo_head)) {
-			moea_attr_clear(pg, PTE_EXEC);
-		} else {
-			was_exec = moea_attr_fetch(pg) & PTE_EXEC;
-		}
-	}
-
 	pte_lo = moea_calc_wimg(VM_PAGE_TO_PHYS(m), pmap_page_get_memattr(m));
 
 	if (prot & VM_PROT_WRITE) {
@@ -1117,22 +1103,14 @@ moea_enter_locked(pmap_t pmap, vm_offset
 	    pte_lo, pvo_flags);
 
 	/*
-	 * Flush the real page from the instruction cache if this page is
-	 * mapped executable and cacheable and was not previously mapped (or
-	 * was not mapped executable).
+	 * Flush the real page from the instruction cache. This has be done
+	 * for all user mappings to prevent information leakage via the
+	 * instruction cache. moea_pvo_enter() returns ENOENT for the first
+	 * mapping for a page.
 	 */
-	if (error == 0 && (pvo_flags & PVO_EXECUTABLE) &&
-	    (pte_lo & PTE_I) == 0 && was_exec == 0) {
-		/*
-		 * Flush the real memory from the cache.
-		 */
+	if (pmap != kernel_pmap && error == ENOENT &&
+	    (pte_lo & (PTE_I | PTE_G)) == 0)
 		moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE);
-		if (pg != NULL)
-			moea_attr_save(pg, PTE_EXEC);
-	}
-
-	/* XXX syncicache always until problems are sorted */
-	moea_syncicache(VM_PAGE_TO_PHYS(m), PAGE_SIZE);
 }
 
 /*
@@ -1454,12 +1432,6 @@ moea_kenter_attr(mmu_t mmu, vm_offset_t 
 		panic("moea_kenter: failed to enter va %#x pa %#x: %d", va,
 		    pa, error);
 
-	/*
-	 * Flush the real memory from the instruction cache.
-	 */
-	if ((pte_lo & (PTE_I | PTE_G)) == 0) {
-		moea_syncicache(pa, PAGE_SIZE);
-	}
 	PMAP_UNLOCK(kernel_pmap);
 }
 

Modified: stable/9/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- stable/9/sys/powerpc/aim/mmu_oea64.c	Fri Apr 20 00:34:15 2012	(r234475)
+++ stable/9/sys/powerpc/aim/mmu_oea64.c	Fri Apr 20 00:45:23 2012	(r234476)
@@ -283,8 +283,6 @@ static struct	pvo_entry *moea64_pvo_find
 /*
  * Utility routines.
  */
-static void		moea64_enter_locked(mmu_t, pmap_t, vm_offset_t,
-			    vm_page_t, vm_prot_t, boolean_t);
 static boolean_t	moea64_query_bit(mmu_t, vm_page_t, u_int64_t);
 static u_int		moea64_clear_bit(mmu_t, vm_page_t, u_int64_t);
 static void		moea64_kremove(mmu_t, vm_offset_t);
@@ -1167,30 +1165,11 @@ moea64_zero_page_idle(mmu_t mmu, vm_page
  * target pmap with the protection requested.  If specified the page
  * will be wired down.
  */
+
 void
 moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, 
     vm_prot_t prot, boolean_t wired)
 {
-
-	LOCK_TABLE_WR();
-	PMAP_LOCK(pmap);
-	moea64_enter_locked(mmu, pmap, va, m, prot, wired);
-	UNLOCK_TABLE_WR();
-	PMAP_UNLOCK(pmap);
-}
-
-/*
- * Map the given physical page at the specified virtual address in the
- * target pmap with the protection requested.  If specified the page
- * will be wired down.
- *
- * The table (write) and pmap must be locked.
- */
-
-static void
-moea64_enter_locked(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m,
-    vm_prot_t prot, boolean_t wired)
-{
 	struct		pvo_head *pvo_head;
 	uma_zone_t	zone;
 	vm_page_t	pg;
@@ -1210,10 +1189,9 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm
 		pvo_flags = PVO_MANAGED;
 	}
 
-	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	KASSERT((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) != 0 ||
 	    VM_OBJECT_LOCKED(m->object),
-	    ("moea64_enter_locked: page %p is not busy", m));
+	    ("moea64_enter: page %p is not busy", m));
 
 	/* XXX change the pvo head for fake pages */
 	if ((m->oflags & VPO_UNMANAGED) != 0) {
@@ -1238,15 +1216,22 @@ moea64_enter_locked(mmu_t mmu, pmap_t pm
 	if (wired)
 		pvo_flags |= PVO_WIRED;
 
+	LOCK_TABLE_WR();
+	PMAP_LOCK(pmap);
 	error = moea64_pvo_enter(mmu, pmap, zone, pvo_head, va,
 	    VM_PAGE_TO_PHYS(m), pte_lo, pvo_flags);
+	PMAP_UNLOCK(pmap);
+	UNLOCK_TABLE_WR();
 
 	/*
 	 * Flush the page from the instruction cache if this page is
 	 * mapped executable and cacheable.
 	 */
-	if ((pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0)
+	if (pmap != kernel_pmap && !(m->aflags & PGA_EXECUTABLE) &&
+	    (pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+		vm_page_aflag_set(m, PGA_EXECUTABLE);
 		moea64_syncicache(mmu, pmap, va, VM_PAGE_TO_PHYS(m), PAGE_SIZE);
+	}
 }
 
 static void
@@ -1307,15 +1292,11 @@ moea64_enter_object(mmu_t mmu, pmap_t pm
 
 	psize = atop(end - start);
 	m = m_start;
-	LOCK_TABLE_WR();
-	PMAP_LOCK(pm);
 	while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
-		moea64_enter_locked(mmu, pm, start + ptoa(diff), m, prot &
+		moea64_enter(mmu, pm, start + ptoa(diff), m, prot &
 		    (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
 		m = TAILQ_NEXT(m, listq);
 	}
-	UNLOCK_TABLE_WR();
-	PMAP_UNLOCK(pm);
 }
 
 void
@@ -1323,12 +1304,8 @@ moea64_enter_quick(mmu_t mmu, pmap_t pm,
     vm_prot_t prot)
 {
 
-	LOCK_TABLE_WR();
-	PMAP_LOCK(pm);
-	moea64_enter_locked(mmu, pm, va, m,
+	moea64_enter(mmu, pm, va, m,
 	    prot & (VM_PROT_READ | VM_PROT_EXECUTE), FALSE);
-	UNLOCK_TABLE_WR();
-	PMAP_UNLOCK(pm);
 }
 
 vm_paddr_t
@@ -1664,12 +1641,6 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_
 	if (error != 0 && error != ENOENT)
 		panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
 		    pa, error);
-
-	/*
-	 * Flush the memory from the instruction cache.
-	 */
-	if ((pte_lo & (LPTE_I | LPTE_G)) == 0)
-		__syncicache((void *)va, PAGE_SIZE);
 }
 
 void
@@ -1900,6 +1871,7 @@ static void
 moea64_pvo_protect(mmu_t mmu,  pmap_t pm, struct pvo_entry *pvo, vm_prot_t prot)
 {
 	uintptr_t pt;
+	struct	vm_page *pg;
 	uint64_t oldlo;
 
 	PMAP_LOCK_ASSERT(pm, MA_OWNED);
@@ -1923,18 +1895,20 @@ moea64_pvo_protect(mmu_t mmu,  pmap_t pm
 	else
 		pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
 
+	pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
+
 	/*
 	 * If the PVO is in the page table, update that pte as well.
 	 */
-	if (pt != -1) {
+	if (pt != -1)
 		MOEA64_PTE_CHANGE(mmu, pt, &pvo->pvo_pte.lpte,
 		    pvo->pvo_vpn);
-		if ((pvo->pvo_pte.lpte.pte_lo & 
-		    (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
-			moea64_syncicache(mmu, pm, PVO_VADDR(pvo),
-			    pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN,
-			    PAGE_SIZE);
-		}
+	if (pm != kernel_pmap && pg != NULL && !(pg->aflags & PGA_EXECUTABLE) &&
+	    (pvo->pvo_pte.lpte.pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
+		if ((pg->oflags & VPO_UNMANAGED) == 0)
+			vm_page_aflag_set(pg, PGA_EXECUTABLE);
+		moea64_syncicache(mmu, pm, PVO_VADDR(pvo),
+		    pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, PAGE_SIZE);
 	}
 
 	/*
@@ -1943,9 +1917,6 @@ moea64_pvo_protect(mmu_t mmu,  pmap_t pm
 	 */
 	if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && 
 	    (oldlo & LPTE_PP) != LPTE_BR && !(prot && VM_PROT_WRITE)) {
-		struct	vm_page *pg;
-
-		pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
 		if (pg != NULL) {
 			if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
 				vm_page_dirty(pg);
@@ -2128,15 +2099,11 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_o
 void
 moea64_remove_all(mmu_t mmu, vm_page_t m)
 {
-	struct  pvo_head *pvo_head;
 	struct	pvo_entry *pvo, *next_pvo;
 	pmap_t	pmap;
 
-	pvo_head = vm_page_to_pvoh(m);
 	LOCK_TABLE_WR();
-	for (pvo = LIST_FIRST(pvo_head); pvo != NULL; pvo = next_pvo) {
-		next_pvo = LIST_NEXT(pvo, pvo_vlink);
-
+	LIST_FOREACH_SAFE(pvo, vm_page_to_pvoh(m), pvo_vlink, next_pvo) {
 		pmap = pvo->pvo_pmap;
 		PMAP_LOCK(pmap);
 		moea64_pvo_remove(mmu, pvo);
@@ -2146,6 +2113,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m
 	if ((m->aflags & PGA_WRITEABLE) && moea64_is_modified(mmu, m))
 		vm_page_dirty(m);
 	vm_page_aflag_clear(m, PGA_WRITEABLE);
+	vm_page_aflag_clear(m, PGA_EXECUTABLE);
 }
 
 /*
@@ -2350,6 +2318,7 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, u
 static void
 moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo)
 {
+	struct	vm_page *pg;
 	uintptr_t pt;
 
 	PMAP_LOCK_ASSERT(pvo->pvo_pmap, MA_OWNED);
@@ -2389,12 +2358,10 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
 	/*
 	 * Update vm about the REF/CHG bits if the page is managed.
 	 */
-	if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED &&
-	    (pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
-		struct	vm_page *pg;
+	pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
 
-		pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN);
-		if (pg != NULL) {
+	if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && pg != NULL) {
+		if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
 			if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG)
 				vm_page_dirty(pg);
 			if (pvo->pvo_pte.lpte.pte_lo & LPTE_REF)
@@ -2402,6 +2369,8 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_
 			if (LIST_EMPTY(vm_page_to_pvoh(pg)))
 				vm_page_aflag_clear(pg, PGA_WRITEABLE);
 		}
+		if (LIST_EMPTY(vm_page_to_pvoh(pg)))
+			vm_page_aflag_clear(pg, PGA_EXECUTABLE);
 	}
 
 	moea64_pvo_entries--;

Modified: stable/9/sys/powerpc/aim/moea64_native.c
==============================================================================
--- stable/9/sys/powerpc/aim/moea64_native.c	Fri Apr 20 00:34:15 2012	(r234475)
+++ stable/9/sys/powerpc/aim/moea64_native.c	Fri Apr 20 00:45:23 2012	(r234476)
@@ -153,13 +153,10 @@ TLBIE(uint64_t vpn) {
 	vpn &= ~(0xffffULL << 48);
 
 #ifdef __powerpc64__
-	sched_pin();
-	__asm __volatile("ptesync");
 	mtx_lock(&tlbie_mutex);
 	__asm __volatile("tlbie %0" :: "r"(vpn) : "memory");
 	mtx_unlock(&tlbie_mutex);
 	__asm __volatile("eieio; tlbsync; ptesync");
-	sched_unpin();
 #else
 	vpn_hi = (uint32_t)(vpn >> 32);
 	vpn_lo = (uint32_t)vpn;
@@ -171,7 +168,6 @@ TLBIE(uint64_t vpn) {
 	    mr %1, %0; \
 	    insrdi %1,%5,1,0; \
 	    mtmsrd %1; isync; \
-	    ptesync; \
 	    \
 	    sld %1,%2,%4; \
 	    or %1,%1,%3; \
@@ -265,7 +261,9 @@ moea64_pte_clear_native(mmu_t mmu, uintp
 	 * As shown in Section 7.6.3.2.3
 	 */
 	pt->pte_lo &= ~ptebit;
+	sched_pin();
 	TLBIE(vpn);
+	sched_unpin();
 }
 
 static void
@@ -295,21 +293,16 @@ moea64_pte_unset_native(mmu_t mmu, uintp
 {
 	struct lpte *pt = (struct lpte *)pt_cookie;
 
-	pvo_pt->pte_hi &= ~LPTE_VALID;
-
-	/* Finish all pending operations */
-	isync();
-
-	/*
-	 * Force the reg & chg bits back into the PTEs.
-	 */
-	SYNC();
-
 	/*
 	 * Invalidate the pte.
 	 */
+	isync();
+	sched_pin();
+	pvo_pt->pte_hi &= ~LPTE_VALID;
 	pt->pte_hi &= ~LPTE_VALID;
+	PTESYNC();
 	TLBIE(vpn);
+	sched_unpin();
 
 	/*
 	 * Save the reg & chg bits.

Modified: stable/9/sys/vm/vm_page.h
==============================================================================
--- stable/9/sys/vm/vm_page.h	Fri Apr 20 00:34:15 2012	(r234475)
+++ stable/9/sys/vm/vm_page.h	Fri Apr 20 00:45:23 2012	(r234476)
@@ -248,9 +248,13 @@ extern struct vpglocks pa_lock[];
  *
  * PGA_WRITEABLE is set exclusively on managed pages by pmap_enter().  When it
  * does so, the page must be VPO_BUSY.
+ *
+ * PGA_EXECUTABLE may be set by pmap routines, and indicates that a page has
+ * at least one executable mapping. It is not consumed by the VM layer.
  */
 #define	PGA_WRITEABLE	0x01		/* page may be mapped writeable */
 #define	PGA_REFERENCED	0x02		/* page has been referenced */
+#define	PGA_EXECUTABLE	0x04		/* page may be mapped executable */
 
 /*
  * Page flags.  If changed at any other time than page allocation or



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201204200045.q3K0jO65014820>