Date: Sun, 9 Apr 2006 19:40:22 GMT From: Olivier Houchard <cognet@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 94863 for review Message-ID: <200604091940.k39JeMpr009803@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=94863 Change 94863 by cognet@cognet on 2006/04/09 19:39:31 Be a bit smarter when promoting pages, and don't do it if different small pages into the larger page have different cache mode. This let us mapping PTE as write-through again, instead of forcing writa-back, however that means we have to demote some kernel superpage, if one of them contains PTEs. Affected files ... .. //depot/projects/superpages/src/sys/arm/arm/pmap.c#9 edit .. //depot/projects/superpages/src/sys/arm/include/pmap.h#8 edit Differences ... ==== //depot/projects/superpages/src/sys/arm/arm/pmap.c#9 (text+ko) ==== @@ -220,6 +220,7 @@ static void pmap_promote_large(pmap_t, vm_offset_t va, reservation_t); static void pmap_demote(pmap_t, vm_offset_t); +static void pmap_demote_large(pmap_t, vm_offset_t); static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); @@ -286,6 +287,11 @@ char *_tmppt; +#ifdef ARM_USE_SMALL_ALLOC +extern struct mtx smallalloc_mtx; +extern vm_offset_t alloc_curaddr; +extern vm_offset_t alloc_firstaddr; +#endif /* * Metadata for L1 translation tables. */ @@ -619,11 +625,9 @@ pte_l2_s_cache_mode = L2_B|L2_C; pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; -#if 0 pte_l1_s_cache_mode_pt = L1_S_C; pte_l2_l_cache_mode_pt = L2_C; pte_l2_s_cache_mode_pt = L2_C; -#endif #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE /* * The XScale core has an enhanced mode where writes that @@ -682,9 +686,6 @@ xscale_use_minidata = 1; #endif - pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode; - pte_l2_l_cache_mode_pt = pte_l2_s_cache_mode; - pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; pte_l2_s_prot_u = L2_S_PROT_U_xscale; pte_l2_s_prot_w = L2_S_PROT_W_xscale; pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; @@ -1099,9 +1100,7 @@ #ifndef PMAP_INCLUDE_PTE_SYNC struct l2_bucket *l2b; pt_entry_t *ptep, pte; -#ifdef ARM_USE_SMALL_ALLOC pd_entry_t *pde; -#endif vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; /* @@ -1112,33 +1111,38 @@ * page tables, we simply fix up the cache-mode here if it's not * correct. */ + pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)]; #ifdef ARM_USE_SMALL_ALLOC - pde = &kernel_pmap->pm_l1->l1_kva[L1_IDX(va)]; - if (l1pte_section_p(*pde) && va < 0xd0000000 && va > virtual_avail) - panic("ouin"); - if (!l1pte_section_p(*pde)) { + if (l1pte_section_p(*pde) && va >= alloc_firstaddr) + return (0); #endif + if (l1pte_section_p(*pde) && (*pde & pte_l1_s_cache_mask) != + pte_l1_s_cache_mode_pt) + pmap_demote_large(pmap_kernel(), va); + else if (l1pte_section_p(*pde)) + return (0); + + l2b = pmap_get_l2_bucket(pmap_kernel(), va); + ptep = &l2b->l2b_kva[l2pte_index(va)]; + if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L && + (*ptep & L2_L_CACHE_MASK) != pte_l2_l_cache_mode_pt) { + pmap_demote(pmap_kernel(), va); l2b = pmap_get_l2_bucket(pmap_kernel(), va); - ptep = &l2b->l2b_kva[l2pte_index(va)]; - if ((*ptep & L2_TYPE_MASK) == L2_TYPE_L) - panic("fuxor"); - pte = *ptep; + } + pte = *ptep; - if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { - /* - * Page tables must have the cache-mode set to - * Write-Thru. - */ - *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; - PTE_SYNC(ptep); - cpu_tlb_flushD_SE(va); - cpu_cpwait(); - } + if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { + /* + * Page tables must have the cache-mode set to + * Write-Thru. + */ + *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; + PTE_SYNC(ptep); + cpu_tlb_flushD_SE(va); + cpu_cpwait(); -#ifdef ARM_USE_SMALL_ALLOC } #endif -#endif memset(mem, 0, L2_TABLE_SIZE_REAL); PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); return (0); @@ -1551,6 +1555,7 @@ int i; pt_entry_t *ptep; vm_paddr_t pa0; + int cachemode; ptep = &l2b->l2b_kva[l2pte_index(va & L2_L_FRAME)]; pa0 = *ptep & L2_L_FRAME; @@ -1574,10 +1579,14 @@ if (*ptep & L2_S_PROT_W) pa0 |= L2_L_PROT_W; /* Let's do it. */ + if ((*ptep & pte_l2_s_cache_mask) == pte_l2_s_cache_mode) + cachemode = pte_l2_l_cache_mode; + else + cachemode = pte_l2_l_cache_mode_pt; for (i = 0; i < 0x10; i++) { pmap_tlb_flushID_SE(pmap, va + i * PAGE_SIZE); ptep[i] = pa0 | L2_L_PROTO | - pte_l2_l_cache_mode; + cachemode; } } @@ -1592,6 +1601,7 @@ struct l1_ttable *l1; uint16_t l1idx; int i; + int cachemode, cachemode2; va0 = va & L1_S_ADDR_MASK; l1idx = L1_IDX(va0); @@ -1614,18 +1624,43 @@ for (i = 1; i < 0x100; i++) { vm_paddr_t pa, pa2; - if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L) + if ((pt[i] & L2_TYPE_MASK) == L2_TYPE_L) { pa = (pt[i] & L2_L_FRAME) + (i & 0xf) * PAGE_SIZE; - else + if ((pt[i] & pte_l2_l_cache_mask) == + pte_l2_l_cache_mode) + cachemode = pte_l1_s_cache_mode; + else + cachemode = pte_l1_s_cache_mode_pt; + } else { pa = pt[i] & L2_S_FRAME; - if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L) + if ((pt[i] & pte_l2_s_cache_mask) == + pte_l2_s_cache_mode) + cachemode = pte_l1_s_cache_mode; + else + cachemode = pte_l1_s_cache_mode_pt; + } + if ((pt[i - 1] & L2_TYPE_MASK) == L2_TYPE_L) { pa2 = (pt[i - 1] & L2_L_FRAME) + ((i - 1) & 0xf) * PAGE_SIZE; - else + if ((pt[i - 1] & pte_l2_l_cache_mask) == + pte_l2_l_cache_mode) + cachemode2 = pte_l1_s_cache_mode; + else + cachemode2 = pte_l1_s_cache_mode_pt; + + } else { pa2 = pt[i - 1] & L2_S_FRAME; + if ((pt[i - 1] & pte_l2_s_cache_mask) == + pte_l2_s_cache_mode) + cachemode2 = pte_l1_s_cache_mode; + else + cachemode2 = pte_l1_s_cache_mode_pt; + } if (pa != pa2 + PAGE_SIZE) /* Nothing much we can do. */ return; + if (cachemode != cachemode2) + return; } #ifdef SP_DEBUG printf("promoting large %x\n", va); @@ -1635,7 +1670,7 @@ pa |= L1_S_PROT_U; if (*pt & L2_S_PROT_W) pa |= L1_S_PROT_W; - *pd = L1_S_PROTO | pa | pte_l1_s_cache_mode | L1_S_DOM(pmap->pm_domain); + *pd = L1_S_PROTO | pa | cachemode | L1_S_DOM(pmap->pm_domain); pmap_free_l2_bucket(pmap, &l2->l2_bucket[L2_BUCKET(l1idx)], 0x100); if (pmap == kernel_pmap) { SLIST_FOREACH(l1, &l1_list, l1_link) { @@ -1645,6 +1680,69 @@ } } +/* + * Special case of pmap_demote(), where we know we want to demote a section + * mapping into large pages, except for the concerned page. + */ + +static void +pmap_demote_large(pmap_t pmap, vm_offset_t va) +{ + pd_entry_t *pd; + pt_entry_t *pt; + struct l2_bucket *l2b; + struct l1_ttable *l1; + vm_offset_t va0; + uint16_t l1idx; + vm_paddr_t pa; + int i; + int mode_l = 0, mode_s = 0; + +#ifdef SP_DEBUG + printf("demoting section mapping at %x\n", va); +#endif + l1idx = L1_IDX(va); + pd = &pmap->pm_l1->l1_kva[l1idx]; + va0 = va & L1_S_ADDR_MASK; + if (pmap == pmap_kernel()) + l2b = pmap_get_l2_bucket(pmap, va0); + else + l2b = pmap_alloc_l2_bucket(pmap, va0, M_NOWAIT); + pa = *pd & L1_S_ADDR_MASK; + if (*pd & L1_S_PROT_U) { + mode_l |= L2_L_PROT_U; + mode_s |= L2_S_PROT_U; + } + if (*pd & L1_S_PROT_W) { + mode_l |= L2_L_PROT_W; + mode_s |= L2_S_PROT_W; + } + if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode) { + mode_l |= pte_l2_l_cache_mode; + mode_s |= pte_l2_s_cache_mode; + } else { + mode_l |= pte_l2_l_cache_mode_pt; + mode_s |= pte_l2_s_cache_mode_pt; + } + pt = &l2b->l2b_kva[l2pte_index(va0)]; + *pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO; + l2b->l2b_occupancy += 0x100; + if (pmap == kernel_pmap) { + SLIST_FOREACH(l1, &l1_list, l1_link) { + l1->l1_kva[l1idx] = *pd; + PTE_SYNC(&l1->l1_kva[l1idx]); + } + } + for (i = 0; i < 0x100; i++, pa++) { + if (va0 + i * PAGE_SIZE != va) + pt[i] = (pa & L2_L_FRAME) | mode_l; + else + pt[i] = pa | mode_s; + } + pmap_tlb_flushID(pmap); + +} + static void pmap_demote(pmap_t pmap, vm_offset_t va) { @@ -1657,6 +1755,7 @@ uint16_t demote_size; vm_paddr_t pa; int i; + int cachemode; #ifdef SP_DEBUG printf("demoting %x\n", va); @@ -1675,6 +1774,10 @@ pa |= L2_S_PROT_U; if (*pd & L1_S_PROT_W) pa |= L2_S_PROT_W; + if ((*pd & pte_l1_s_cache_mask) == pte_l1_s_cache_mode) + cachemode = pte_l2_s_cache_mode; + else + cachemode = pte_l2_s_cache_mode_pt; pt = &l2b->l2b_kva[l2pte_index(va0)]; *pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO; l2b->l2b_occupancy += 0x100; @@ -1696,10 +1799,14 @@ pa |= L2_S_PROT_U; if (*pt & L2_L_PROT_W) pa |= L2_S_PROT_W; + if ((*pt & pte_l2_l_cache_mask) == pte_l2_l_cache_mode) + cachemode = pte_l2_s_cache_mode; + else + cachemode = pte_l2_s_cache_mode_pt; } pa |= L2_S_PROTO; for (i = 0; i < demote_size; i++, pa += PAGE_SIZE) - pt[i] = (pa) | pte_l2_s_cache_mode; + pt[i] = (pa) | cachemode; pmap_tlb_flushID(pmap); } @@ -2661,11 +2768,6 @@ * (physical) address starting relative to 0] */ #define PMAP_STATIC_L2_SIZE 16 -#ifdef ARM_USE_SMALL_ALLOC -extern struct mtx smallalloc_mtx; -extern vm_offset_t alloc_curaddr; -extern vm_offset_t alloc_firstaddr; -#endif void pmap_bootstrap(vm_offset_t firstaddr, vm_offset_t lastaddr, struct pv_addr *l1pt) ==== //depot/projects/superpages/src/sys/arm/include/pmap.h#8 (text+ko) ==== @@ -337,7 +337,7 @@ #define PMAP_NEEDS_PTE_SYNC 1 #define PMAP_INCLUDE_PTE_SYNC #elif (ARM_MMU_SA1 == 0) -#if 1 +#if defined(CPU_ARM9) && !defined(ARM9_CACHE_WRITE_THROUGH) #define PMAP_NEEDS_PTE_SYNC 1 #define PMAP_INCLUDE_PTE_SYNC #else
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200604091940.k39JeMpr009803>