Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 4 Oct 2005 13:13:08 GMT
From:      Olivier Houchard <cognet@FreeBSD.org>
To:        Perforce Change Reviews <perforce@freebsd.org>
Subject:   PERFORCE change 84785 for review
Message-ID:  <200510041313.j94DD8MP001118@repoman.freebsd.org>

next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=84785

Change 84785 by cognet@cognet on 2005/10/04 13:13:08

	Superpages bits for arm.

Affected files ...

.. //depot/projects/superpages/src/sys/arm/arm/pmap.c#3 edit
.. //depot/projects/superpages/src/sys/arm/include/pmap.h#3 edit

Differences ...

==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#3 (text+ko) ====

@@ -1,4 +1,4 @@
-/* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */
+//* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */
 /*-
  * Copyright 2004 Olivier Houchard.
  * Copyright 2003 Wasabi Systems, Inc.
@@ -168,6 +168,7 @@
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
+#include <vm/vm_reserve.h>
 #include <vm/vm_extern.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
@@ -210,10 +211,16 @@
 static int		pmap_clearbit(struct vm_page *, u_int);
 
 static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t);
-static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t);
+static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t, int flags);
 static void		pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int);
 static vm_offset_t	kernel_pt_lookup(vm_paddr_t);
 
+static void		pmap_promote_small(pmap_t, struct l2_bucket *, 
+    vm_offset_t, reservation_t, int);
+static void		pmap_promote_large(pmap_t, vm_offset_t va,
+    reservation_t);
+static void		pmap_demote(pmap_t, vm_offset_t);
+
 static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1");
 
 vm_offset_t avail_end;		/* PA of last available physical page */
@@ -910,7 +917,7 @@
  * the bucket/page in the meantime.
  */
 static struct l2_bucket *
-pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va)
+pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va, int flags)
 {
 	struct l2_dtable *l2;
 	struct l2_bucket *l2b;
@@ -946,7 +953,7 @@
 		 * No L2 page table has been allocated. Chances are, this
 		 * is because we just allocated the l2_dtable, above.
 		 */
-		ptep = (void*)uma_zalloc(l2zone, M_NOWAIT);
+		ptep = (void*)uma_zalloc(l2zone, flags);
 		l2b->l2b_phys = vtophys(ptep);
 		if (ptep == NULL) {
 			/*
@@ -1105,10 +1112,14 @@
 	 */
 #ifdef ARM_USE_SMALL_ALLOC
 	pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)];
+	if (l1pte_section_p(*pde) && va < 0xd0000000 && va > virtual_avail)
+		panic("ouin");
 	if (!l1pte_section_p(*pde)) {
 #endif
 		l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 		ptep = &l2b->l2b_kva[l2pte_index(va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+			panic("fuxor");
 		pte = *ptep;
 
 		if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) {
@@ -1454,8 +1465,18 @@
 
 			pv->pv_flags |= PVF_NC;
 
+			if (l1pte_section_p(
+			    pm->pm_l1->l1_kva[L1_IDX(pv->pv_va)])) {
+				pmap_demote(pv->pv_pmap, pv->pv_va);
+				pv->pv_flags |= PVF_DEMOTED;
+			}
 			l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+			
 			ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+			if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+				pmap_demote(pv->pv_pmap, pv->pv_va);
+				pv->pv_flags |= PVF_DEMOTED;
+			}
 			pte = *ptep & ~L2_S_CACHE_MASK;
 
 			if ((va != pv->pv_va || pm != pv->pv_pmap) &&
@@ -1509,10 +1530,177 @@
 
 			*ptep = pte;
 			PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
+			if (pv->pv_flags & PVF_DEMOTED) {
+				pmap_promote_small(pv->pv_pmap, l2b, pv->pv_va,
+				    NULL, 0);
+				pmap_promote_large(pv->pv_pmap, pv->pv_va,
+				    NULL);
+				pv->pv_flags &= ~PVF_DEMOTED;
+			}
 		}
 	}
 }
 
+/* XXX: misleading, promote_small actually promotes to large pages. */
+static void
+pmap_promote_small(pmap_t pmap, struct l2_bucket *l2b, vm_offset_t va,
+    reservation_t reserv, int off)
+{
+	int i;
+	pt_entry_t *ptep;
+	vm_paddr_t pa0;
+	
+	if (!reserv)
+		return;
+	ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)];
+	pa0 = *ptep & L2_L_FRAME;
+	if ((*ptep & L2_S_FRAME) != pa0)
+		return;
+	if (reserv != NULL && ((pa0 - off * PAGE_SIZE)
+	    != VM_PAGE_TO_PHYS(reserv->first_page)))
+		return;
+	/* First check it is eligible for promotion. */
+	for (i = 1; i < 0x10; i++) {
+		if (ptep[i] != (ptep[i - 1] + PAGE_SIZE)) {
+			/* There's no way we can do it. */
+			return;
+		}
+		if ((ptep[i] & pte_l2_s_cache_mask) == 
+		    pte_l2_s_cache_mode_pt || 
+		    (ptep[i - 1] & pte_l2_s_cache_mask) ==
+		    pte_l2_s_cache_mode_pt)
+			panic("fuck");
+	}
+#ifdef SP_DEBUG
+	printf("promoting small %x\n", va);
+#endif
+	if (*ptep & L2_S_PROT_U)
+		pa0 |= L2_L_PROT_U;
+	if (*ptep & L2_S_PROT_W)
+		pa0 |= L2_L_PROT_W;
+	/* Let's do it. */
+	for (i = 0; i < 0x10; i++)
+		ptep[i] = pa0 | L2_L_PROTO |
+		    pte_l2_l_cache_mode;
+	pmap_tlb_flushID(pmap);
+}
+
+static void
+pmap_promote_large(pmap_t pmap, vm_offset_t va, reservation_t reserv)
+{
+	pd_entry_t *pd;
+	pt_entry_t *pt;
+	vm_offset_t va0;
+	vm_paddr_t pa;
+	struct l2_dtable *l2;
+	uint16_t l1idx;
+	int i;
+	
+	if (!reserv)
+		return;
+	va0 = va & L1_S_ADDR_MASK;
+	l1idx = L1_IDX(va0);
+	pd = &pmap->pm_l1->l1_kva[l1idx];
+	KASSERT(pd != NULL, ("NO pd_entry in pmap_promote_large"));
+	l2 = pmap->pm_l2[L2_IDX(l1idx)];
+	KASSERT(l2 != NULL, ("No l2 bucket in pmap_promote_large"));
+	pt = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva;
+	if (l2pte_index(va0) != 0)
+		panic("rate");
+	KASSERT(pt != NULL, ("No pt in pmap_promote_large"));
+	if ((*pt & L2_TYPE_MASK) == L2_TYPE_L)
+		pa = *pt & L2_L_FRAME;
+	else
+		pa = *pt & L2_S_FRAME;
+	if (pa != (*pt & L1_S_FRAME))
+		return;
+	if (reserv != NULL && pa != VM_PAGE_TO_PHYS(reserv->first_page))
+		return;
+	for (i = 1; i < 0x100; i++) {
+		vm_paddr_t pa, pa2;
+
+		if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L)
+			pa = (pt[i] & L2_L_FRAME) + (i & 0xf) * PAGE_SIZE;
+		else
+			pa = pt[i] & L2_S_FRAME;
+		if ((pt[i - 1] & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt
+		    || (pt[i] & pte_l2_s_cache_mask) == pte_l2_s_cache_mode_pt)
+			panic("fuck2");
+		if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L)
+			pa2 = (pt[i - 1] & L2_L_FRAME) +
+			    ((i - 1) & 0xf) * PAGE_SIZE;
+		else
+			pa2 = pt[i - 1] & L2_S_FRAME;
+		if (pa != pa2 + PAGE_SIZE)
+			/* Nothing much we can do. */
+			return;
+	}
+#ifdef SP_DEBUG
+	printf("promoting large %x\n", va);
+#endif
+	/* Now free the unused pts. */
+	if (*pt & L2_S_PROT_U)
+		pa |= L1_S_PROT_U;
+	if (*pt & L2_S_PROT_W)
+		pa |= L1_S_PROT_W;
+	*pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain);
+	bzero(pt, 0x100 * sizeof(*pt));
+	pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100);
+	pmap_tlb_flushID(pmap);
+}
+
+static void
+pmap_demote(pmap_t pmap, vm_offset_t va)
+{
+	pd_entry_t *pd;
+	pt_entry_t *pt;
+	struct l2_bucket *l2b;
+	vm_offset_t va0;
+	uint16_t l1idx;
+	uint16_t demote_size;
+	vm_paddr_t pa;
+	int i;
+
+#ifdef SP_DEBUG
+	printf("demoting %x\n", va);
+#endif
+	l1idx = L1_IDX(va);
+	pd = &pmap->pm_l1->l1_kva[l1idx];
+	if (l1pte_section_p(*pd)) {
+		va0 = va & L1_S_ADDR_MASK;
+		if (pmap == pmap_kernel())
+			l2b = pmap_get_l2_bucket(pmap, va0);
+		else
+			l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT);
+		demote_size = 0x100;
+		pa = *pd & L1_S_ADDR_MASK;
+		if (*pd & L1_S_PROT_U)
+			pa |= L2_S_PROT_U;
+		if (*pd & L1_S_PROT_W)
+			pa |= L2_S_PROT_W;
+		pt = &l2b->l2b_kva[l2pte_index(va0)];
+		*pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO;
+		l2b->l2b_occupancy += 0x100;
+	} else {
+		va0 = va & L2_L_FRAME;
+		l2b = pmap_get_l2_bucket(pmap, va0);
+		demote_size = 0x10;
+		pt = &l2b->l2b_kva[l2pte_index(va0)];
+		if ((*pt & L2_TYPE_MASK) != L2_TYPE_L)
+			return;
+		pa = *pt & L2_L_FRAME;
+		if (*pt & L2_L_PROT_U)
+			pa |= L2_S_PROT_U;
+		if (*pt & L2_L_PROT_W)
+			pa |= L2_S_PROT_W;
+	}
+	pa |= L2_S_PROTO;
+	for (i = 0; i < demote_size; i++, pa += PAGE_SIZE)
+		pt[i] = (pa) | pte_l2_s_cache_mode;
+	pmap_tlb_flushID(pmap);
+}
+
+
 /*
  * Modify pte bits for all ptes corresponding to the given physical address.
  * We use `maskbits' rather than `clearbits' because we're always passing
@@ -1559,9 +1747,17 @@
 		pmap_acquire_pmap_lock(pm);
 #endif
 
+
+		if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(va)])) {
+			pmap_demote(pm, va);
+			pv->pv_flags |= PVF_DEMOTED;
+		}
 		l2b = pmap_get_l2_bucket(pm, va);
-
 		ptep = &l2b->l2b_kva[l2pte_index(va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+			pmap_demote(pm, va);
+			pv->pv_flags |= PVF_DEMOTED;
+		}
 		npte = opte = *ptep;
 
 		if (maskbits & (PVF_WRITE|PVF_MOD) &&
@@ -1675,6 +1871,11 @@
 			if (PV_BEEN_REFD(oflags))
 				pmap_tlb_flushD_SE(pm, pv->pv_va);
 		}
+		if (pv->pv_flags & PVF_DEMOTED) {
+			pmap_promote_small(pm, l2b, va, NULL, 0);
+			pmap_promote_large(pm, va, NULL);
+			pv->pv_flags &= ~PVF_DEMOTED;
+		}
 
 #if 0
 		pmap_release_pmap_lock(pm);
@@ -1713,8 +1914,6 @@
 pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm,
     vm_offset_t va, u_int flags)
 {
-
-
 	pve->pv_pmap = pm;
 	pve->pv_va = va;
 	pve->pv_flags = flags;
@@ -1981,6 +2180,19 @@
 	pmap_acquire_pmap_lock(pm);
 #endif
 	l1idx = L1_IDX(va);
+	pl1pd = &pm->pm_l1->l1_kva[l1idx];
+	if (l1pte_section_p(*pl1pd)) {
+		if (user && !(*pl1pd & L1_S_PROT_U))
+			goto out;
+		if (ftype == VM_PROT_WRITE && !(*pl1pd & L1_S_PROT_W))
+			goto out;
+		if ((*pl1pd & L1_S_DOM_MASK) != L1_S_DOM(pm->pm_domain)) {
+			*pl1pd &= ~L1_S_DOM_MASK;
+			*pl1pd |= L1_S_DOM(pm->pm_domain);
+			rv = 1;
+			goto out;
+		}
+	}
 
 	/*
 	 * If there is no l2_dtable for this address, then the process
@@ -2011,12 +2223,15 @@
 	/*
 	 * Catch a userland access to the vector page mapped at 0x0
 	 */
-	if (user && (pte & L2_S_PROT_U) == 0)
+	if (user && (((pte & L2_TYPE_MASK) == L2_TYPE_L &&
+	    (pte & L2_L_PROT_U) == 0) || (pte & L2_S_PROT_U) == 0))
 		goto out;
 
-	pa = l2pte_pa(pte);
+	pa = (pte & L2_TYPE_MASK) == L2_TYPE_L ? 
+	    ((pte & L2_L_FRAME) | (va & L2_L_OFFSET)) : l2pte_pa(pte);
 
-	if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) {
+	if ((ftype & VM_PROT_WRITE) && (((pte & L2_TYPE_MASK) == L2_TYPE_L &&
+	    (pte & L2_L_PROT_W) == 0) || (pte & L2_S_PROT_W) == 0)) {
 		/*
 		 * This looks like a good candidate for "page modified"
 		 * emulation...
@@ -2025,15 +2240,13 @@
 		struct vm_page *pg;
 
 		/* Extract the physical address of the page */
-		if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) {
+		if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL)
 			goto out;
-		}
 		/* Get the current flags for this page. */
 
 		pv = pmap_find_pv(pg, pm, va);
-		if (pv == NULL) {
+		if (pv == NULL)
 			goto out;
-		}
 
 		/*
 		 * Do the flags say this page is writable? If not then it
@@ -2042,9 +2255,8 @@
 		 * PTE. Now we know a write has occurred we can correct this
 		 * and also set the modified bit
 		 */
-		if ((pv->pv_flags & PVF_WRITE) == 0) {
+		if ((pv->pv_flags & PVF_WRITE) == 0)
 			goto out;
-		}
 
 		if (pmap_track_modified(pv->pv_va)) {
 			pg->md.pvh_attrs |= PVF_REF | PVF_MOD;
@@ -2059,7 +2271,12 @@
 		 * changing. We've already set the cacheable bits based on
 		 * the assumption that we can write to this page.
 		 */
-		*ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W;
+		if ((pte & L2_TYPE_MASK) == L2_TYPE_L)
+			*ptep = (pte & ~L2_TYPE_MASK) | L2_L_PROTO |
+			    L2_L_PROT_W;
+		else
+			*ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO |
+			    L2_S_PROT_W;
 		PTE_SYNC(ptep);
 		rv = 1;
 	} else
@@ -2088,7 +2305,6 @@
 		pg->md.pvh_attrs |= PVF_REF;
 		pv->pv_flags |= PVF_REF;
 
-
 		*ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO;
 		PTE_SYNC(ptep);
 		rv = 1;
@@ -2212,7 +2428,7 @@
 	pv_entry_high_water = 9 * (pv_entry_max / 10);
 	l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor,
 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE);
-	uma_prealloc(l2zone, 4096);
+	/*uma_prealloc(l2zone, 4096);*/
 	l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR,
 	    UMA_ZONE_VM | UMA_ZONE_NOFREE);
@@ -2823,8 +3039,10 @@
 {
 	struct pv_entry *pv, *npv;
 	struct l2_bucket *l2b = NULL;
+	pt_entry_t *pt;
+	pd_entry_t *pd;
+	vm_paddr_t pa;
 	vm_page_t m;
-	pt_entry_t *pt;
 	
 	vm_page_lock_queues();
 	for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) {
@@ -2832,6 +3050,9 @@
 			npv = TAILQ_NEXT(pv, pv_plist);
 			continue;
 		}
+		pd = &pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+		if (l1pte_section_p(*pd))
+			pmap_demote(pmap, pv->pv_va);
 		if (pv->pv_flags & PVF_WIRED) {
 			/* The page is wired, cannot remove it now. */
 			npv = TAILQ_NEXT(pv, pv_plist);
@@ -2841,7 +3062,10 @@
 		l2b = pmap_get_l2_bucket(pmap, pv->pv_va);
 		KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages"));
 		pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
-		m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK);
+		if ((*pt & L2_TYPE_MASK) == L2_TYPE_L)
+			pmap_demote(pmap, pv->pv_va);
+		pa = *pt & L2_S_FRAME;
+		m = PHYS_TO_VM_PAGE(pa);
 		*pt = 0;
 		PTE_SYNC(pt);
 		npv = TAILQ_NEXT(pv, pv_plist);
@@ -3014,6 +3238,7 @@
 	TAILQ_FOREACH(pv, &m->md.pv_list, pv_list)
 	    pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE);
 }
+
 /*
  * Add a list of wired pages to the kva
  * this routine is only used for temporary
@@ -3084,6 +3309,8 @@
 
 	if (!pmap_get_pde_pte(pmap, addr, &pde, &pte))
 		return (FALSE);
+	if (!pte)
+		return (FALSE);
 	if (*pte == 0)
 		return (TRUE);
 	return (FALSE);
@@ -3154,6 +3381,7 @@
 pmap_remove_all(vm_page_t m)
 {
 	pv_entry_t pv;
+	pd_entry_t *pd;
 	pt_entry_t *ptep, pte;
 	struct l2_bucket *l2b;
 	boolean_t flush = FALSE;
@@ -3177,9 +3405,15 @@
 		if (flush == FALSE && (pv->pv_pmap == curpm ||
 		    pv->pv_pmap == pmap_kernel()))
 			flush = TRUE;
+		pd = &pv->pv_pmap->pm_l1->l1_kva[L1_IDX(pv->pv_va)];
+		if (l1pte_section_p(*pd))
+			pmap_demote(pv->pv_pmap, pv->pv_va);
 		l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va);
+		
 		KASSERT(l2b != NULL, ("No l2 bucket"));
 		ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+			pmap_demote(pv->pv_pmap, pv->pv_va);
 		pte = *ptep;
 		*ptep = 0;
 		PTE_SYNC_CURRENT(pv->pv_pmap, ptep);
@@ -3214,6 +3448,7 @@
 	vm_offset_t next_bucket;
 	u_int flags;
 	int flush;
+	int demoted = 0;
 
 	if ((prot & VM_PROT_READ) == 0) {
 		mtx_lock(&Giant);
@@ -3243,10 +3478,25 @@
 
 	vm_page_lock_queues();
 	while (sva < eva) {
+		demoted = 0;
 		next_bucket = L2_NEXT_BUCKET(sva);
 		if (next_bucket > eva)
 			next_bucket = eva;
 
+		if (l1pte_section_p(pm->pm_l1->l1_kva[L1_IDX(sva)])) {
+			if ((sva & L1_S_OFFSET) == 0 &&
+			    sva + L1_S_SIZE < eva) {
+				/* Change the whole 1MB superpage. */
+				pm->pm_l1->l1_kva[L1_IDX(sva)] &= ~
+				    L1_S_PROT_W;
+				flush++;
+				sva += L1_S_SIZE;
+				continue;
+			}
+			pmap_demote(pm, sva);
+			l2b = pmap_get_l2_bucket(pm, sva);
+			demoted = 1;
+		}
 		l2b = pmap_get_l2_bucket(pm, sva);
 		if (l2b == NULL) {
 			sva = next_bucket;
@@ -3256,6 +3506,11 @@
 		ptep = &l2b->l2b_kva[l2pte_index(sva)];
 
 		while (sva < next_bucket) {
+			demoted = 0;
+			if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) {
+				pmap_demote(pm, sva);
+				demoted = 1;
+			}
 			if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) {
 				struct vm_page *pg;
 				u_int f;
@@ -3285,6 +3540,10 @@
 					pmap_tlb_flushD_SE(pm, sva);
 			}
 
+			if (demoted) {
+				pmap_promote_small(pm, l2b, sva, NULL, 0);
+				pmap_promote_large(pm, sva, NULL);
+			}
 			sva += PAGE_SIZE;
 			ptep++;
 		}
@@ -3322,13 +3581,40 @@
     boolean_t wired)
 {
 	struct l2_bucket *l2b = NULL;
-	struct vm_page *opg;
+	struct vm_page *opg = NULL;
 	struct pv_entry *pve = NULL;
 	pt_entry_t *ptep, npte, opte;
+	pd_entry_t l1pd;
 	u_int nflags;
 	u_int oflags;
 	vm_paddr_t pa;
 
+	l1pd = pmap->pm_l1->l1_kva[L1_IDX(va)];
+	if (l1pte_section_p(l1pd) &&
+	    ((l1pd & L1_S_FRAME) != (VM_PAGE_TO_PHYS(m) & L1_S_FRAME) ||
+	     (((VM_PAGE_TO_PHYS(m) & L2_S_FRAME) & L2_S_OFFSET) !=
+	      (va & L2_S_OFFSET)) ||
+	     ((l1pd & L1_S_PROT_U) != 0) != (pmap != pmap_kernel() && va !=
+		vector_page) || ((l1pd & L1_S_PROT_W) != 0) !=
+	     ((prot & VM_PROT_WRITE) != 0))) {
+		/* There's already a superpage at this address, demote. */
+		pmap_demote(pmap, va);
+	} else if (l1pte_section_p(l1pd)) {
+		pmap_free_l2_bucket(pmap, l2b, 0);
+		opg = m;
+		l2b = NULL;
+	}
+	
+	if (opg == NULL) {
+		if (pmap == pmap_kernel()) {
+			l2b = pmap_get_l2_bucket(pmap, va);
+			if (!l2b)
+				l2b = pmap_grow_l2_bucket(pmap, va);
+		} else
+			l2b = pmap_alloc_l2_bucket(pmap, va, M_NOWAIT);
+		KASSERT(l2b != NULL,
+		    ("pmap_enter: failed to allocate l2 bucket"));
+	}
 	vm_page_lock_queues();
 	if (va == vector_page) {
 		pa = systempage.pv_pa;
@@ -3345,31 +3631,40 @@
 	PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, "
 	    "wired = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, wired));
 	    
-	if (pmap == pmap_kernel()) {
-		l2b = pmap_get_l2_bucket(pmap, va);
-		if (l2b == NULL)
-			l2b = pmap_grow_l2_bucket(pmap, va);
-	} else
-		l2b = pmap_alloc_l2_bucket(pmap, va);
-		KASSERT(l2b != NULL,
-		    ("pmap_enter: failed to allocate l2 bucket"));
-	ptep = &l2b->l2b_kva[l2pte_index(va)];
+	if (l2b) {
+		ptep = &l2b->l2b_kva[l2pte_index(va)];
 		    
-	opte = *ptep;
-	npte = pa;
-	oflags = 0;
+		opte = *ptep;
+		npte = pa; 
+		oflags = 0;
+	} else {
+		npte = oflags = opte = 0;
+		ptep = NULL;
+	}
 	if (opte) {
+		if ((opte & L2_TYPE_MASK) == L2_TYPE_L) {
+			if (((opte & L2_L_FRAME) | (va & L2_L_OFFSET)) ==
+			    VM_PAGE_TO_PHYS(m) &&
+			    (((opte & L2_L_PROT_W) != 0) ==
+			    (prot & VM_PROT_WRITE) != 0) &&
+			    (((opte & L2_L_PROT_U) == 0) ==
+			     (pmap == pmap_kernel() || va == vector_page)))
+				opg = m;
+			else {
+				pmap_demote(pmap, va);
+				opte = *ptep;
+			}
+		}
 		/*
 		 * There is already a mapping at this address.
 		 * If the physical address is different, lookup the
 		 * vm_page.
 		 */
-		if (l2pte_pa(opte) != pa)
+		if (!opg && l2pte_pa(opte) != pa)
 			opg = PHYS_TO_VM_PAGE(l2pte_pa(opte));
-		else
+		else if (!opg)
 			opg = m;
-	} else
-		opg = NULL;
+	}
 
 	if ((prot & (VM_PROT_ALL)) ||
 	    (!m || m->md.pvh_attrs & PVF_REF)) {
@@ -3492,7 +3787,7 @@
 	/*
 	 * Keep the stats up to date
 	 */
-	if (opte == 0) {
+	if (opte == 0 && l2b) {
 		l2b->l2b_occupancy++;
 		pmap->pm_stats.resident_count++;
 	} 
@@ -3502,7 +3797,7 @@
 	 * If this is just a wiring change, the two PTEs will be
 	 * identical, so there's no need to update the page table.
 	 */
-	if (npte != opte) {
+	if (l2b && npte != opte && ((opte & L2_TYPE_MASK) != L2_TYPE_L)) {
 		boolean_t is_cached = pmap_is_current(pmap);
 
 		*ptep = npte;
@@ -3539,6 +3834,17 @@
 
 
 		pmap_vac_me_harder(m, pmap, va);
+		if ((va < VM_MAXUSER_ADDRESS ||
+		    m->object == kernel_object || 
+		    (m->object == kmem_object && FALSE)) && m->reserv) {
+			if (m->reserv->refcnt > 0 &&
+			    m->reserv->refcnt != SP_LARGE &&
+			    (m->reserv->refcnt % SMALL) == 0)
+				pmap_promote_small(pmap, l2b, va, m->reserv,
+				    m->reserv->refcnt - SMALL);
+			else if (m->reserv->refcnt == SP_LARGE)
+				pmap_promote_large(pmap, va, m->reserv);
+		}
 	}
 	vm_page_unlock_queues();
 }
@@ -3579,15 +3885,19 @@
 void
 pmap_change_wiring(pmap_t pmap, vm_offset_t va, boolean_t wired)
 {
+#if 0
 	struct l2_bucket *l2b;
 	pt_entry_t *ptep, pte;
+#endif
 	vm_page_t pg;
 
+#if 0
 	l2b = pmap_get_l2_bucket(pmap, va);
 	KASSERT(l2b, ("No l2b bucket in pmap_change_wiring"));
 	ptep = &l2b->l2b_kva[l2pte_index(va)];
 	pte = *ptep;
-	pg = PHYS_TO_VM_PAGE(l2pte_pa(pte));
+#endif
+	pg = PHYS_TO_VM_PAGE(pmap_extract(pmap, va));
 	if (pg) 
 		pmap_modify_pv(pg, pmap, va, PVF_WIRED, wired);
 }
@@ -3626,10 +3936,6 @@
 	l1pd = *pl1pd;
 
 	if (l1pte_section_p(l1pd)) {
-		/*
-		 * These should only happen for pmap_kernel()
-		 */
-		KASSERT(pm == pmap_kernel(), ("huh"));
 		pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
 	} else {
 		/*
@@ -3685,10 +3991,6 @@
 
 	vm_page_lock_queues();
 	if (l1pte_section_p(l1pd)) {
-		/*
-		 * These should only happen for pmap_kernel()
-		 */
-		KASSERT(pmap == pmap_kernel(), ("huh"));
 		pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET);
 		if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) {
 			m = PHYS_TO_VM_PAGE(pa);
@@ -3845,8 +4147,22 @@
 
 		l2b = pmap_get_l2_bucket(pm, sva);
 		if (l2b == NULL) {
-			sva = next_bucket;
-			continue;
+			pd_entry_t *pd = &pm->pm_l1->l1_kva[L1_IDX(sva)];
+			if (l1pte_section_p(*pd)) {
+				/* We can just remove the superpage. */
+				if (0 && (sva == (sva & L1_S_ADDR_MASK)) &&
+				    (sva + 0x100000 < eva)) {
+					*pd = 0;
+					sva = sva + 0x100000;
+					continue;
+				} else {
+					pmap_demote(pm, sva);
+					l2b = pmap_get_l2_bucket(pm, sva);
+				}
+			} else {
+				sva = next_bucket;
+				continue;
+			}
 		}
 
 		ptep = &l2b->l2b_kva[l2pte_index(sva)];
@@ -3857,6 +4173,10 @@
 			pt_entry_t pte;
 			vm_paddr_t pa;
 
+			if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+			/* 64KB superpage. */
+				pmap_demote(pm, sva);
+
 			pte = *ptep;
 
 			if (pte == 0) {
@@ -4017,7 +4337,6 @@
 	    _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0)
 		return;
 
-
 	/*
 	 * Hook in the page, zero it, and purge the cache for that
 	 * zeroed page. Invalidate the TLB as needed.
@@ -4085,7 +4404,11 @@
 
 		l2b = pmap_get_l2_bucket(pmap_kernel(), va);
 
+		if (!l2b)
+			panic("fuck2");
 		sptep = ptep = &l2b->l2b_kva[l2pte_index(va)];
+		if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L)
+			panic("bordel");
 
 		while (va < next_bucket) {
 			pte = *ptep;
@@ -4826,4 +5149,3 @@
 
 	return (NULL);
 }
-

==== //depot/projects/superpages/src/sys/arm/include/pmap.h#3 (text+ko) ====

@@ -479,6 +479,7 @@
 #define	PVF_EXEC	0x10		/* mapping is executable */
 #define	PVF_UNC		0x20		/* mapping is 'user' non-cacheable */
 #define	PVF_KNC		0x40		/* mapping is 'kernel' non-cacheable */
+#define PVF_DEMOTED	0x80		/* Page was demoted. */
 #define	PVF_NC		(PVF_UNC|PVF_KNC)
 
 void vector_page_setprot(int);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200510041313.j94DD8MP001118>