Date: Tue, 9 May 2006 00:17:58 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 96842 for review Message-ID: <200605090017.k490Hw6x071443@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=96842 Change 96842 by kmacy@kmacy_storage:sun4v_rwbuf on 2006/05/09 00:17:16 use direct area for referencing TSB at TL < 2 once the TSB is installed clean up prologue to tsb miss handler - greatly simplifying don't always clear TSB in invalidate_range and invalidate_page - allowing us to pre-load the TSB in places make more aggressive use of membars when updating the TSB Affected files ... .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/pmap.h#12 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#11 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#50 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#48 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#13 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#9 edit Differences ... ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/pmap.h#12 (text+ko) ==== @@ -102,8 +102,8 @@ void pmap_bootstrap(vm_offset_t ekva); vm_paddr_t pmap_kextract(vm_offset_t va); -void pmap_invalidate_page(pmap_t pmap, vm_offset_t va); -void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva); +void pmap_invalidate_page(pmap_t pmap, vm_offset_t va, int cleartsb); +void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int cleartsb); void pmap_invalidate_all(pmap_t pmap); void pmap_scrub_pages(vm_paddr_t pa, int64_t size); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#11 (text+ko) ==== @@ -32,6 +32,8 @@ void tsb_set_tte(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx); +void tsb_set_tte_real(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx); + tte_t tsb_get_tte(struct hv_tsb_info *tsb, vm_offset_t va); tte_t tsb_lookup_tte(vm_offset_t va, uint64_t context); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#50 (text+ko) ==== @@ -1393,67 +1393,41 @@ ! %g3==flag bits, TSB (RA) ! %g4==fault type,entry tag ! %g5==tag -! %g6==context,hash size, temp +! %g6==context ! %g7 temp ENTRY(tsb_miss_handler) ldxa [%g1 + %g7]ASI_REAL, %g6 ! load in the context - rdpr %tl, %g7 ! need to use real addresses - -#ifdef notyet - rdpr %cansave, %g1 -#else - mov 0, %g1 -#endif - - brz,pn %g1, 0f - nop - save -0: - cmp %g7, 1 ! for tl > 1 - bne,pn %xcc, 2f - nop - mov ASI_LDTD_N, %g3 - wr %g0, ASI_N, %asi - GET_PCPU_SCRATCH - cmp %g6, %g0 ! kernel? - be,pn %xcc, 1f - nop GET_HASH_SCRATCH_USER(%g2) GET_TSB_SCRATCH_USER(%g4) - brz,pn %g1, 4f - nop - ba,pt %xcc, 5f - nop -1: + + brnz,pn %g6, 2f + nop GET_HASH_SCRATCH_KERNEL(%g2) GET_TSB_SCRATCH_KERNEL(%g4) - brz,pn %g1, 4f - nop - ba,pt %xcc, 5f - nop 2: - mov ASI_LDTD_REAL, %g3 - wr %g0, ASI_REAL, %asi - GET_PCPU_PHYS_SCRATCH(%g4) - cmp %g6, %g0 ! kernel? - be,pn %xcc, 3f - nop - GET_HASH_PHYS_SCRATCH_USER(%g4, %g2) - GET_TSB_SCRATCH_USER(%g4) - brz,pn %g1, 4f - nop - ba,pt %xcc, 5f - nop -3: + rdpr %tl, %g1 ! need to use real addresses? + mov ASI_LDTD_N, %g3 + wr %g0, ASI_N, %asi + dec %g1 + GET_PCPU_SCRATCH - GET_HASH_PHYS_SCRATCH_KERNEL(%g4, %g2) - GET_TSB_SCRATCH_KERNEL(%g4) - brnz,pt %g1, 5f + brz,pt %g1, 3f ! for tl == 1 nop -4: + sethi %uhi(VM_MIN_DIRECT_ADDRESS), %g1 + wr %g0, ASI_REAL, %asi + sllx %g1, 32, %g1 + mov ASI_LDTD_REAL, %g3 + andn %g2, %g1, %g2 + andn %g4, %g1, %g4 + andn %g7, %g1, %g7 +3: +#ifdef notyet + rdpr %cansave, %g1 + /* XXX use save operation if %g1 > 0 and tl == 1 */ +#endif rdpr %tl, %g1 dec %g1 sll %g1, RW_SHIFT, %g1 @@ -1461,7 +1435,6 @@ add PCPU_REG, %g1, %g1 SAVE_LOCALS_ASI(%g1) mov 0, %g1 ! cansave is 0 -5: ! %g1 == %cansave ! %g2 == hash scratch value @@ -1588,10 +1561,10 @@ sllx %g3, TTE_SHIFT, %g3 ! masked byte offset add %g3, %l3, %g3 ! TTE RA - mov 8, %l4 #ifdef PMAP_DEBUG - ldda [%g3]ASI_LDTD_REAL, %l2 + ldxa [%g3]%asi, %l2 + ldxa [%g3 + 8]%asi, %l3 cmp %l3, %l7 bne,pt %xcc, 12f cmp %l2, %l6 @@ -1612,12 +1585,10 @@ #endif 12: #endif - stxa %g0, [%g3 + %l4]ASI_REAL ! invalidate data - membar #StoreStore - stxa %l6, [%g3]ASI_REAL ! store tag - membar #StoreStore - stxa %l7, [%g3 + %l4]ASI_REAL ! store data - stxa %l7, [%g2 + 8]%asi ! update TTE with ref bit + stxa %g0, [%g3 + 8]%asi ! invalidate data + stxa %l6, [%g3]%asi ! store tag + stxa %l7, [%g3 + 8]%asi ! store data + stxa %l7, [%g2 + 8]%asi ! update TTE with ref bit membar #StoreLoad THE_LOCK_EXIT(%l5, %l0, %l7) ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#48 (text+ko) ==== @@ -332,7 +332,7 @@ vm_page_dirty(m); } - pmap_invalidate_page(pmap, va); + pmap_invalidate_page(pmap, va, TRUE); TAILQ_REMOVE(&pmap->pm_pvlist, pv, pv_plist); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) @@ -639,7 +639,7 @@ for (i = 0; i < KSTACK_PAGES; i++) { pa = kstack0_phys + i * PAGE_SIZE; va = kstack0 + i * PAGE_SIZE; - tsb_set_tte(&kernel_td[TSB8K_INDEX], va, + tsb_set_tte_real(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0); } /* @@ -674,7 +674,7 @@ va = translations[i].om_start + off; pa = TTE_GET_PA(translations[i].om_tte) + off; tsb_assert_invalid(&kernel_td[TSB8K_INDEX], va); - tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa | + tsb_set_tte_real(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0); } } @@ -687,7 +687,7 @@ pa = PAGE_SIZE_4M; for (i = 0; phys_avail[i + 2] != 0; i += 2) for (; pa < phys_avail[i + 1]; pa += PAGE_SIZE_4M) { - tsb_set_tte(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa), + tsb_set_tte_real(&kernel_td[TSB4M_INDEX], TLB_PHYS_TO_DIRECT(pa), pa | TTE_KERNEL | VTD_4M, 0); } @@ -966,6 +966,8 @@ otte_data = tte_hash_update(pmap->pm_hash, va, tte_data | TTE_MINFLAGS); + tsb_set_tte(&pmap->pm_tsb, va, tte_data|TTE_MINFLAGS, pmap->pm_context); + invlva = FALSE; if ((otte_data & ~(VTD_W|VTD_REF)) != tte_data) { if (otte_data & VTD_V) { @@ -988,7 +990,7 @@ } if (invlva) - pmap_invalidate_page(pmap, va); + pmap_invalidate_page(pmap, va, FALSE); sched_unpin(); PMAP_UNLOCK(pmap); @@ -1229,11 +1231,12 @@ #endif void -pmap_invalidate_page(pmap_t pmap, vm_offset_t va) +pmap_invalidate_page(pmap_t pmap, vm_offset_t va, int cleartsb) { spinlock_enter(); - tsb_clear_tte(&pmap->pm_tsb, va); + if (cleartsb == TRUE) + tsb_clear_tte(&pmap->pm_tsb, va); DPRINTF("pmap_invalidate_page(va=0x%lx)\n", va); invlpg(va, pmap->pm_context); @@ -1247,7 +1250,7 @@ } void -pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) +pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int cleartsb) { vm_offset_t tva; #ifdef SMP @@ -1256,14 +1259,12 @@ spinlock_enter(); - if (pmap != kernel_pmap) - DPRINTF("pmap_invalidate_range(sva=%lx, eva=%lx)\n", sva, eva); - - if ((((eva - sva) >> PAGE_SHIFT) < MAX_TSB_CLEARS) || - (pmap->pm_context == 0)) { - tsb_clear_range(&pmap->pm_tsb, sva, eva); - } else { - tsb_clear(&pmap->pm_tsb); + if (cleartsb == TRUE) { + if ((((eva - sva) >> PAGE_SHIFT) < MAX_TSB_CLEARS) || + (pmap->pm_context == 0)) + tsb_clear_range(&pmap->pm_tsb, sva, eva); + else + tsb_clear(&pmap->pm_tsb); } /* XXX */ @@ -1342,6 +1343,7 @@ pmap_kenter(vm_offset_t va, vm_paddr_t pa) { tte_hash_update(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K); + tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0); } /* @@ -1402,7 +1404,7 @@ start += PAGE_SIZE; } - pmap_invalidate_range(kernel_pmap, sva, va); + pmap_invalidate_range(kernel_pmap, sva, va, FALSE); *virt = va; return (sva); @@ -1569,7 +1571,7 @@ sched_unpin(); if (anychanged) - pmap_invalidate_range(pmap, sva, eva); + pmap_invalidate_range(pmap, sva, eva, TRUE); vm_page_unlock_queues(); PMAP_UNLOCK(pmap); @@ -1591,7 +1593,7 @@ va += PAGE_SIZE; m++; } - pmap_invalidate_range(kernel_pmap, sva, va); + pmap_invalidate_range(kernel_pmap, sva, va, FALSE); } /* @@ -1608,7 +1610,7 @@ pmap_kremove(va); va += PAGE_SIZE; } - pmap_invalidate_range(kernel_pmap, sva, va); + pmap_invalidate_range(kernel_pmap, sva, va, TRUE); } @@ -1661,7 +1663,7 @@ sched_unpin(); vm_page_unlock_queues(); if (invlva) - pmap_invalidate_range(pmap, start, end); + pmap_invalidate_range(pmap, start, end, TRUE); PMAP_UNLOCK(pmap); @@ -1711,7 +1713,7 @@ vm_page_dirty(m); } - pmap_invalidate_page(pv->pv_pmap, pv->pv_va); + pmap_invalidate_page(pv->pv_pmap, pv->pv_va, TRUE); TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); m->md.pv_list_count--; ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#13 (text+ko) ==== @@ -51,6 +51,8 @@ #include <machine/mmu.h> #include <machine/tte.h> #include <machine/tsb.h> +#include <machine/vmparam.h> +#include <machine/tlb.h> CTASSERT(sizeof(tte_t) == sizeof(uint64_t)); #define TSB_MASK(tsb) ((tsb->hvtsb_ntte) - 1) @@ -92,7 +94,7 @@ hvtsb->hvtsb_pa = VM_PAGE_TO_PHYS(m); tsb_pages = hvtsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT); - *scratchval = hvtsb->hvtsb_pa | tsb_pages; + *scratchval = TLB_PHYS_TO_DIRECT(hvtsb->hvtsb_pa) | tsb_pages; return vtophys(hvtsb); } @@ -127,7 +129,7 @@ } void -tsb_set_tte(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx) +tsb_set_tte_real(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx) { vm_paddr_t tsb_store_pa; uint64_t tsb_index, tsb_shift, tte_tag; @@ -152,6 +154,30 @@ store_real(tsb_store_pa + sizeof(uint64_t), tte_data); } + +void +tsb_set_tte(hv_tsb_info_t *tsb, vm_offset_t va, uint64_t tte_data, uint64_t ctx) +{ + + uint64_t tsb_index, tsb_shift, tte_tag; + tte_t *entry; + + tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz); + tsb_index = (va >> tsb_shift) & TSB_MASK(tsb); + entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t)); + tte_tag = (ctx << TTARGET_CTX_SHIFT) | (va >> TTARGET_VA_SHIFT); + tte_data &= ~VTD_V; + /* store new value with valid bit cleared + * to avoid invalid intermediate value; + */ + *(entry + 1) = tte_data; + tte_data |= VTD_V; + *(entry) = tte_tag; + *(entry + 1) = tte_data; + membar(Sync); +} + + void tsb_clear(hv_tsb_info_t *tsb) { @@ -161,40 +187,60 @@ void tsb_clear_tte(hv_tsb_info_t *tsb, vm_offset_t va) { - vm_paddr_t tsb_store_pa; + tte_t *entry; uint64_t tsb_index, tsb_shift; tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz); tsb_index = (va >> tsb_shift) & TSB_MASK(tsb); - tsb_store_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t); + entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t)); + + *(entry) = 0; + *(entry + 1) = 0; - store_real(tsb_store_pa, 0); - store_real(tsb_store_pa + sizeof(uint64_t), 0); + membar(Sync); } void tsb_clear_range(hv_tsb_info_t *tsb, vm_offset_t sva, vm_offset_t eva) { - vm_paddr_t tsb_store_pa; vm_offset_t tva; uint64_t tsb_index, tsb_shift, tsb_mask; - + tte_t *entry; + tsb_mask = TSB_MASK(tsb); tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz); for (tva = sva; tva < eva; tva += PAGE_SIZE) { tsb_index = (tva >> tsb_shift) & tsb_mask; - tsb_store_pa = tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t); - - store_real(tsb_store_pa, 0); - store_real(tsb_store_pa + sizeof(uint64_t), 0); + entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t)); + *(entry) = 0; + *(entry + 1) = 0; } + membar(Sync); } tte_t tsb_get_tte(hv_tsb_info_t *tsb, vm_offset_t va) { + tte_t *entry; + uint64_t tsb_index, tsb_shift, tte_tag, tte_data; + + tsb_shift = TTE_PAGE_SHIFT(tsb->hvtsb_idxpgsz); + tsb_index = (va >> tsb_shift) & TSB_MASK(tsb); + entry = (tte_t *)TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa + 2*tsb_index*sizeof(uint64_t)); + tte_tag = *(entry); + tte_data = *(entry + 1); + + if ((tte_tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M)) + return tte_data; + + return (0UL); +} +#if 0 +tte_t +tsb_get_tte_real(hv_tsb_info_t *tsb, vm_offset_t va) +{ vm_paddr_t tsb_load_pa; uint64_t tsb_index, tsb_shift, tte_tag, tte_data; @@ -213,6 +259,7 @@ return (0UL); } +#endif tte_t tsb_lookup_tte(vm_offset_t va, uint64_t ctx) @@ -238,7 +285,7 @@ { uint64_t tsb_pages, tsb_scratch; tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT); - tsb_scratch = tsb->hvtsb_pa | tsb_pages; + tsb_scratch = TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa) | tsb_pages; set_tsb_kernel_scratchpad(tsb_scratch); membar(Sync); @@ -250,7 +297,7 @@ { uint64_t tsb_pages, tsb_scratch; tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT); - tsb_scratch = tsb->hvtsb_pa | tsb_pages; + tsb_scratch = TLB_PHYS_TO_DIRECT(tsb->hvtsb_pa) | tsb_pages; set_tsb_user_scratchpad(tsb_scratch); membar(Sync); return tsb_scratch; ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#9 (text+ko) ==== @@ -81,7 +81,7 @@ if (otte_data & VTD_W) vm_page_dirty(m); - pmap_invalidate_page(pv->pv_pmap, pv->pv_va); + pmap_invalidate_page(pv->pv_pmap, pv->pv_va, TRUE); } @@ -112,7 +112,7 @@ otte_data = tte_hash_clear_bits(pmap->pm_hash, va, flags); if (otte_data & flags) - pmap_invalidate_page(pmap, va); + pmap_invalidate_page(pmap, va, TRUE); } void
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200605090017.k490Hw6x071443>