Date: Fri, 19 Mar 2010 16:15:11 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r205337 - in projects/ppc64/sys/powerpc: aim include powerpc Message-ID: <201003191615.o2JGFBxk000295@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Fri Mar 19 16:15:11 2010 New Revision: 205337 URL: http://svn.freebsd.org/changeset/base/205337 Log: Provide a long list of segment mappings for each process, and change the kernel mappings to be (a) calculated instead of looked up, and (b) cached per-CPU. This gives SMP a fighting chance at working, and should allow the use of more than 16 GB of RAM. It also allows the use of more than 16 GB of VA space per process, and allows some minor speed and correctness improvements in a few places. 32-bit kernels are probably very broken on this branch at the moment. This will be fixed later. More breakage should not be forthcoming -- this is the last major change to the memory management code. Modified: projects/ppc64/sys/powerpc/aim/machdep.c projects/ppc64/sys/powerpc/aim/mmu_oea64.c projects/ppc64/sys/powerpc/aim/slb.c projects/ppc64/sys/powerpc/aim/trap.c projects/ppc64/sys/powerpc/aim/trap_subr64.S projects/ppc64/sys/powerpc/include/pcpu.h projects/ppc64/sys/powerpc/include/pmap.h projects/ppc64/sys/powerpc/include/slb.h projects/ppc64/sys/powerpc/powerpc/genassym.c Modified: projects/ppc64/sys/powerpc/aim/machdep.c ============================================================================== --- projects/ppc64/sys/powerpc/aim/machdep.c Fri Mar 19 16:09:57 2010 (r205336) +++ projects/ppc64/sys/powerpc/aim/machdep.c Fri Mar 19 16:15:11 2010 (r205337) @@ -738,7 +738,10 @@ kdb_cpu_set_singlestep(void) void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz) { - +#ifdef __powerpc64__ +/* Copy the SLB contents from the current CPU */ +memcpy(pcpu->pc_slb, PCPU_GET(slb), sizeof(pcpu->pc_slb)); +#endif } void Modified: projects/ppc64/sys/powerpc/aim/mmu_oea64.c ============================================================================== --- projects/ppc64/sys/powerpc/aim/mmu_oea64.c Fri Mar 19 16:09:57 2010 (r205336) +++ projects/ppc64/sys/powerpc/aim/mmu_oea64.c Fri Mar 19 16:15:11 2010 (r205337) @@ -160,6 +160,7 @@ __FBSDID("$FreeBSD$"); #define MOEA_DEBUG #define TODO panic("%s: not implemented", __func__); +void moea64_release_vsid(uint64_t vsid); uintptr_t moea64_get_unique_vsid(void); static __inline register_t @@ -183,21 +184,14 @@ cntlzd(volatile register_t a) { struct mtx tlbie_mutex; static __inline void -TLBIE(pmap_t pmap, vm_offset_t va) { +TLBIE(uint64_t vpn) { #ifndef __powerpc64__ register_t vpn_hi, vpn_lo; register_t msr; register_t scratch; #endif - uint64_t vpn; - /* - * Compute the virtual page number we wish to invalidate. - */ - - vpn = (uint64_t)(va & ADDR_PIDX); - if (pmap != NULL) - vpn |= (va_to_vsid(pmap,va) << 28); + vpn <<= ADDR_PIDX_SHFT; vpn &= ~(0xffffULL << 48); mtx_lock_spin(&tlbie_mutex); @@ -257,6 +251,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) { ((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK))) #define PVO_PTEGIDX_SET(pvo, i) \ ((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID)) +#define PVO_VSID(pvo) ((pvo)->pvo_vpn >> 16) #define MOEA_PVO_CHECK(pvo) @@ -347,6 +342,7 @@ SYSCTL_INT(_machdep, OID_AUTO, moea64_pv &moea64_pvo_remove_calls, 0, ""); vm_offset_t moea64_scratchpage_va[2]; +uint64_t moea64_scratchpage_vpn[2]; struct lpte *moea64_scratchpage_pte[2]; struct mtx moea64_scratchpage_mtx; @@ -572,7 +568,7 @@ moea64_pte_synch(struct lpte *pt, struct } static __inline void -moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit) +moea64_pte_clear(struct lpte *pt, uint64_t vpn, u_int64_t ptebit) { ASSERT_TABLE_LOCK(); @@ -580,7 +576,7 @@ moea64_pte_clear(struct lpte *pt, pmap_t * As shown in Section 7.6.3.2.3 */ pt->pte_lo &= ~ptebit; - TLBIE(pmap,va); + TLBIE(vpn); } static __inline void @@ -603,7 +599,7 @@ moea64_pte_set(struct lpte *pt, struct l } static __inline void -moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va) +moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { ASSERT_TABLE_LOCK(); pvo_pt->pte_hi &= ~LPTE_VALID; @@ -617,7 +613,7 @@ moea64_pte_unset(struct lpte *pt, struct * Invalidate the pte. */ pt->pte_hi &= ~LPTE_VALID; - TLBIE(pmap,va); + TLBIE(vpn); /* * Save the reg & chg bits. @@ -627,16 +623,14 @@ moea64_pte_unset(struct lpte *pt, struct } static __inline void -moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va) +moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn) { /* * Invalidate the PTE */ - moea64_pte_unset(pt, pvo_pt, pmap, va); + moea64_pte_unset(pt, pvo_pt, vpn); moea64_pte_set(pt, pvo_pt); - if (pmap == kernel_pmap) - isync(); } static __inline uint64_t @@ -708,6 +702,9 @@ static void moea64_cpu_bootstrap(mmu_t mmup, int ap) { int i = 0; + #ifdef __powerpc64__ + struct slb *slb = PCPU_GET(slb); + #endif /* * Initialize segment registers and MMU @@ -723,12 +720,11 @@ moea64_cpu_bootstrap(mmu_t mmup, int ap) slbia(); for (i = 0; i < 64; i++) { - if (!(kernel_pmap->pm_slb[i].slbe & SLBE_VALID)) + if (!(slb[i].slbe & SLBE_VALID)) continue; __asm __volatile ("slbmte %0, %1" :: - "r"(kernel_pmap->pm_slb[i].slbv), - "r"(kernel_pmap->pm_slb[i].slbe)); + "r"(slb[i].slbv), "r"(slb[i].slbe)); } #else for (i = 0; i < 16; i++) @@ -796,6 +792,7 @@ moea64_add_ofw_mappings(mmu_t mmup, phan } } +#ifdef __powerpc64__ static void moea64_probe_large_page(void) { @@ -822,6 +819,32 @@ moea64_probe_large_page(void) } static void +moea64_bootstrap_slb_prefault(vm_offset_t va, int large) +{ + struct slb *cache; + struct slb entry; + uint64_t esid, slbe; + uint64_t i; + + cache = PCPU_GET(slb); + esid = va >> ADDR_SR_SHFT; + slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; + + for (i = 0; i < 64; i++) { + if (cache[i].slbe == (slbe | i)) + return; + } + + entry.slbe = slbe; + entry.slbv = (esid | KERNEL_VSID_BIT) << SLBV_VSID_SHIFT; + if (large) + entry.slbv |= SLBV_L; + + slb_insert(kernel_pmap, cache, &entry); +} +#endif + +static void moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { @@ -836,6 +859,7 @@ moea64_setup_direct_map(mmu_t mmup, vm_o DISABLE_TRANS(msr); if (hw_direct_map) { + #ifdef __powerpc64__ PMAP_LOCK(kernel_pmap); for (i = 0; i < pregions_sz; i++) { for (pa = pregions[i].mr_start; pa < pregions[i].mr_start + @@ -854,14 +878,6 @@ moea64_setup_direct_map(mmu_t mmup, vm_o pregions[i].mr_start + pregions[i].mr_size) pte_lo |= LPTE_G; - /* - * Allocate a new SLB entry to make sure it is - * for large pages. - */ - if (va_to_slb_entry(kernel_pmap, pa) == NULL) - allocate_vsid(kernel_pmap, pa >> ADDR_SR_SHFT, - 1 /* large */); - moea64_pvo_enter(kernel_pmap, moea64_upvo_zone, &moea64_pvo_kunmanaged, pa, pa, pte_lo, PVO_WIRED | PVO_LARGE | @@ -869,6 +885,7 @@ moea64_setup_direct_map(mmu_t mmup, vm_o } } PMAP_UNLOCK(kernel_pmap); + #endif } else { size = moea64_pteg_count * sizeof(struct lpteg); off = (vm_offset_t)(moea64_pteg_table); @@ -1076,8 +1093,8 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t */ #ifdef __powerpc64__ for (i = 0; i < 64; i++) { - kernel_pmap->pm_slb[i].slbv = 0; - kernel_pmap->pm_slb[i].slbe = 0; + pcpup->pc_slb[i].slbv = 0; + pcpup->pc_slb[i].slbe = 0; } #else for (i = 0; i < 16; i++) @@ -1101,9 +1118,9 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t */ if (!ofw_real_mode) { + #ifndef __powerpc64__ moea64_pinit(mmup, &ofw_pmap); - #ifndef __powerpc64__ for (i = 0; i < 16; i++) ofw_pmap.pm_sr[i] = kernel_pmap->pm_sr[i]; #endif @@ -1148,6 +1165,14 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS; /* + * Map the entire KVA range into the SLB. We must not fault there. + */ + #ifdef __powerpc64__ + for (va = virtual_avail; va < virtual_end; va += SEGMENT_LENGTH) + moea64_bootstrap_slb_prefault(va, 0); + #endif + + /* * Figure out how far we can extend virtual_end into segment 16 * without running into existing mappings. Segment 16 is guaranteed * to contain neither RAM nor devices (at least on Apple hardware), @@ -1189,6 +1214,9 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t LPTE_NOEXEC, 0); pt.pte_hi |= LPTE_LOCKED; + moea64_scratchpage_vpn[i] = (vsid << 16) | + ((moea64_scratchpage_va[i] & ADDR_PIDX) >> + ADDR_PIDX_SHFT); ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i], 0); pteidx = moea64_pte_insert(ptegidx, &pt); if (pt.pte_hi & LPTE_HID) @@ -1245,23 +1273,22 @@ moea64_bootstrap(mmu_t mmup, vm_offset_t } /* - * Activate a user pmap. The pmap must be activated before it's address + * Activate a user pmap. The pmap must be activated before its address * space can be accessed in any way. */ void moea64_activate(mmu_t mmu, struct thread *td) { - pmap_t pm, pmr; + pmap_t pm; - /* - * Load all the data we need up front to encourage the compiler to - * not issue any loads while we have interrupts disabled below. - */ pm = &td->td_proc->p_vmspace->vm_pmap; - pmr = pm->pmap_phys; - pm->pm_active |= PCPU_GET(cpumask); - PCPU_SET(curpmap, pmr); + + #ifdef __powerpc64__ + PCPU_SET(userslb, pm->pm_slb); + #else + PCPU_SET(curpmap, pm->pmap_phys); + #endif } void @@ -1271,7 +1298,11 @@ moea64_deactivate(mmu_t mmu, struct thre pm = &td->td_proc->p_vmspace->vm_pmap; pm->pm_active &= ~(PCPU_GET(cpumask)); + #ifdef __powerpc64__ + PCPU_SET(userslb, NULL); + #else PCPU_SET(curpmap, NULL); + #endif } void @@ -1310,7 +1341,7 @@ void moea64_set_scratchpage_pa(int which mtx_assert(&moea64_scratchpage_mtx, MA_OWNED); moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID; - TLBIE(kernel_pmap, moea64_scratchpage_va[which]); + TLBIE(moea64_scratchpage_vpn[which]); moea64_scratchpage_pte[which]->pte_lo &= ~(LPTE_WIMG | LPTE_RPGN); @@ -1742,7 +1773,9 @@ moea64_remove_write(mmu_t mmu, vm_page_t lo |= pvo->pvo_pte.lpte.pte_lo; pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG; moea64_pte_change(pt, &pvo->pvo_pte.lpte, - pvo->pvo_pmap, PVO_VADDR(pvo)); + pvo->pvo_vpn); + if (pvo->pvo_pmap == kernel_pmap) + isync(); } } UNLOCK_TABLE(); @@ -1971,13 +2004,20 @@ moea64_get_unique_vsid(void) { panic("%s: out of segments",__func__); } +#ifdef __powerpc64__ void moea64_pinit(mmu_t mmu, pmap_t pmap) { + PMAP_LOCK_INIT(pmap); + + SPLAY_INIT(&pmap->pm_slbtree); + pmap->pm_slb = slb_alloc_user_cache(); +} +#else +void +moea64_pinit(mmu_t mmu, pmap_t pmap) int i; - #ifndef __powerpc64__ register_t hash; - #endif PMAP_LOCK_INIT(pmap); @@ -1987,18 +2027,6 @@ moea64_pinit(mmu_t mmu, pmap_t pmap) else pmap->pmap_phys = pmap; - #ifdef __powerpc64__ - /* - * 64-bit PowerPC uses lazy segment allocation, so NULL - * all the segment entries for now. - */ - for (i = 0; i < sizeof(pmap->pm_slb)/sizeof(pmap->pm_slb[0]); i++) { - pmap->pm_slb[i].slbv = 0; - pmap->pm_slb[i].slbe = 0; - } - - #else - /* * Allocate some segment registers for this pmap. */ @@ -2006,9 +2034,8 @@ moea64_pinit(mmu_t mmu, pmap_t pmap) for (i = 0; i < 16; i++) pmap->pm_sr[i] = VSID_MAKE(i, hash); - - #endif } +#endif /* * Initialize the pmap associated with process 0. @@ -2070,8 +2097,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_ * If the PVO is in the page table, update that pte as well. */ if (pt != NULL) { - moea64_pte_change(pt, &pvo->pvo_pte.lpte, - pvo->pvo_pmap, PVO_VADDR(pvo)); + moea64_pte_change(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); if ((pvo->pvo_pte.lpte.pte_lo & (LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) { moea64_syncicache(pm, sva, @@ -2113,7 +2139,7 @@ moea64_qremove(mmu_t mmu, vm_offset_t va } } -static __inline void +void moea64_release_vsid(uint64_t vsid) { int idx, mask; @@ -2133,9 +2159,8 @@ moea64_release(mmu_t mmu, pmap_t pmap) * Free segment registers' VSIDs */ #ifdef __powerpc64__ - int i; - for (i = 0; i < sizeof(pmap->pm_slb)/sizeof(pmap->pm_slb[0]); i++) - moea64_release_vsid(pmap->pm_slb[i].slbv); + free_vsids(pmap); + slb_free_user_cache(pmap->pm_slb); #else if (pmap->pm_sr[0] == 0) panic("moea64_release"); @@ -2368,6 +2393,8 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z moea64_pvo_entries++; pvo->pvo_vaddr = va; + pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT) + | (vsid << 16); pvo->pvo_pmap = pm; LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink); pvo->pvo_vaddr &= ~ADDR_POFF; @@ -2416,6 +2443,15 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t z UNLOCK_TABLE(); +#ifdef __powerpc64__ + /* + * Make sure all our bootstrap mappings are in the SLB as soon + * as virtual memory is switched on. + */ + if (!pmap_bootstrapped) + moea64_bootstrap_slb_prefault(va, flags & PVO_LARGE); +#endif + return (first ? ENOENT : 0); } @@ -2431,8 +2467,7 @@ moea64_pvo_remove(struct pvo_entry *pvo, LOCK_TABLE(); pt = moea64_pvo_to_pte(pvo, pteidx); if (pt != NULL) { - moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap, - PVO_VADDR(pvo)); + moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn); PVO_PTEGIDX_CLR(pvo); } else { moea64_pte_overflow--; @@ -2500,18 +2535,18 @@ moea64_pvo_find_va(pmap_t pm, vm_offset_ int ptegidx; uint64_t vsid; #ifdef __powerpc64__ - struct slb *slb; + struct slb slb; - slb = va_to_slb_entry(pm, va); - if (slb == NULL) /* The page is not mapped if the segment isn't */ + /* The page is not mapped if the segment isn't */ + if (va_to_slb_entry(pm, va, &slb) != 0) return NULL; - vsid = (slb->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT; - if (slb->slbv & SLBV_L) + vsid = (slb.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT; + if (slb.slbv & SLBV_L) va &= ~moea64_large_page_mask; else va &= ~ADDR_POFF; - ptegidx = va_to_pteg(vsid, va, slb->slbv & SLBV_L); + ptegidx = va_to_pteg(vsid, va, slb.slbv & SLBV_L); #else va &= ~ADDR_POFF; vsid = va_to_vsid(pm, va); @@ -2543,7 +2578,7 @@ moea64_pvo_to_pte(const struct pvo_entry int ptegidx; uint64_t vsid; - vsid = va_to_vsid(pvo->pvo_pmap, PVO_VADDR(pvo)); + vsid = PVO_VSID(pvo); ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo), pvo->pvo_vaddr & PVO_LARGE); pteidx = moea64_pvo_pte_index(pvo, ptegidx); @@ -2724,7 +2759,7 @@ moea64_clear_bit(vm_page_t m, u_int64_t moea64_pte_synch(pt, &pvo->pvo_pte.lpte); if (pvo->pvo_pte.lpte.pte_lo & ptebit) { count++; - moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit); + moea64_pte_clear(pt, pvo->pvo_vpn, ptebit); } } rv |= pvo->pvo_pte.lpte.pte_lo; Modified: projects/ppc64/sys/powerpc/aim/slb.c ============================================================================== --- projects/ppc64/sys/powerpc/aim/slb.c Fri Mar 19 16:09:57 2010 (r205336) +++ projects/ppc64/sys/powerpc/aim/slb.c Fri Mar 19 16:15:11 2010 (r205337) @@ -25,95 +25,160 @@ */ #include <sys/param.h> -#include <sys/kdb.h> +#include <sys/kernel.h> +#include <sys/lock.h> +#include <sys/mutex.h> #include <sys/proc.h> #include <sys/systm.h> +#include <sys/tree.h> #include <vm/vm.h> #include <vm/pmap.h> +#include <vm/uma.h> +#include <vm/vm_map.h> +#include <machine/md_var.h> +#include <machine/pmap.h> #include <machine/vmparam.h> uintptr_t moea64_get_unique_vsid(void); +void moea64_release_vsid(uint64_t vsid); -struct slb * -va_to_slb_entry(pmap_t pm, vm_offset_t va) +struct slbcontainer { + struct slb slb; + SPLAY_ENTRY(slbcontainer) slb_node; +}; + +static int slb_compare(struct slbcontainer *a, struct slbcontainer *b); +static void slb_zone_init(void *); + +SPLAY_PROTOTYPE(slb_tree, slbcontainer, slb_node, slb_compare); +SPLAY_GENERATE(slb_tree, slbcontainer, slb_node, slb_compare); + +uma_zone_t slb_zone; +uma_zone_t slb_cache_zone; + +SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL); + +int +va_to_slb_entry(pmap_t pm, vm_offset_t va, struct slb *slb) { - uint64_t slbe, i; + struct slbcontainer cont, *found; + uint64_t esid; - slbe = (uintptr_t)va >> ADDR_SR_SHFT; - slbe = (slbe << SLBE_ESID_SHIFT) | SLBE_VALID; + esid = (uintptr_t)va >> ADDR_SR_SHFT; + slb->slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; - for (i = 0; i < sizeof(pm->pm_slb)/sizeof(pm->pm_slb[0]); i++) { - if (pm->pm_slb[i].slbe == (slbe | i)) - return &pm->pm_slb[i]; + if (pm == kernel_pmap) { + /* Set kernel VSID to ESID | KERNEL_VSID_BIT */ + slb->slbv = (esid | KERNEL_VSID_BIT) << SLBV_VSID_SHIFT; + + /* Figure out if this is a large-page mapping */ + if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) { + /* + * XXX: If we have set up a direct map, assumes + * all physical memory is mapped with large pages. + */ + if (mem_valid(va, 0) == 0) + slb->slbv |= SLBV_L; + } + + return (0); } - /* XXX: Have a long list for processes mapping more than 16 GB */ + PMAP_LOCK_ASSERT(pm, MA_OWNED); - return (NULL); + cont.slb.slbe = slb->slbe; + found = SPLAY_FIND(slb_tree, &pm->pm_slbtree, &cont); + + if (found == NULL) + return (-1); + + slb->slbv = found->slb.slbv; + return (0); } uint64_t va_to_vsid(pmap_t pm, vm_offset_t va) { - struct slb *entry; + struct slb entry; - entry = va_to_slb_entry(pm, va); + /* Shortcut kernel case: VSID = ESID | KERNEL_VSID_BIT */ + if (pm == kernel_pmap) + return (((uintptr_t)va >> ADDR_SR_SHFT) | KERNEL_VSID_BIT); /* * If there is no vsid for this VA, we need to add a new entry * to the PMAP's segment table. - * - * XXX We assume (for now) that we are not mapping large pages. */ - if (entry == NULL) + if (va_to_slb_entry(pm, va, &entry) != 0) return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0)); - return ((entry->slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT); + return ((entry.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT); } uint64_t allocate_vsid(pmap_t pm, uint64_t esid, int large) { uint64_t vsid; - struct slb slb_entry; + struct slbcontainer *slb_entry, kern_entry; + struct slb *prespill; - vsid = moea64_get_unique_vsid(); + prespill = NULL; - slb_entry.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; - slb_entry.slbv = vsid << SLBV_VSID_SHIFT; + if (pm == kernel_pmap) { + vsid = esid | KERNEL_VSID_BIT; + slb_entry = &kern_entry; + prespill = PCPU_GET(slb); + } else { + vsid = moea64_get_unique_vsid(); + slb_entry = uma_zalloc(slb_zone, M_NOWAIT); + + if (slb_entry == NULL) + panic("Could not allocate SLB mapping!"); + + prespill = pm->pm_slb; + } + + slb_entry->slb.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID; + slb_entry->slb.slbv = vsid << SLBV_VSID_SHIFT; if (large) - slb_entry.slbv |= SLBV_L; + slb_entry->slb.slbv |= SLBV_L; + + if (pm != kernel_pmap) { + PMAP_LOCK_ASSERT(pm, MA_OWNED); + SPLAY_INSERT(slb_tree, &pm->pm_slbtree, slb_entry); + } /* * Someone probably wants this soon, and it may be a wired * SLB mapping, so pre-spill this entry. */ - slb_insert(pm, &slb_entry, 1); + if (prespill != NULL) + slb_insert(pm, prespill, &slb_entry->slb); return (vsid); } -#ifdef NOTYET /* We don't have a back-up list. Spills are a bad idea. */ /* Lock entries mapping kernel text and stacks */ #define SLB_SPILLABLE(slbe) \ (((slbe & SLBE_ESID_MASK) < VM_MIN_KERNEL_ADDRESS && \ - (slbe & SLBE_ESID_MASK) > SEGMENT_LENGTH) || \ + (slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \ (slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS) -#else -#define SLB_SPILLABLE(slbe) 0 -#endif - void -slb_insert(pmap_t pm, struct slb *slb_entry, int prefer_empty) +slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry) { uint64_t slbe, slbv; int i, j, to_spill; + /* + * Note: no locking is necessary in this function because all slbcaches + * are either for the current thread or per-CPU. + */ + to_spill = -1; slbv = slb_entry->slbv; slbe = slb_entry->slbe; @@ -124,31 +189,104 @@ slb_insert(pmap_t pm, struct slb *slb_en if (pm == kernel_pmap && i == USER_SR) continue; - if (!(pm->pm_slb[i].slbe & SLBE_VALID)) { + if (!(slbcache[i].slbe & SLBE_VALID)) { to_spill = i; break; } if (to_spill < 0 && (pm != kernel_pmap || - SLB_SPILLABLE(pm->pm_slb[i].slbe))) { + SLB_SPILLABLE(slbcache[i].slbe))) to_spill = i; - if (!prefer_empty) - break; - } } if (to_spill < 0) panic("SLB spill on ESID %#lx, but no available candidates!\n", (slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT); - pm->pm_slb[to_spill].slbv = slbv; - pm->pm_slb[to_spill].slbe = slbe | (uint64_t)to_spill; + slbcache[to_spill].slbv = slbv; + slbcache[to_spill].slbe = slbe | (uint64_t)to_spill; + /* If it is for this CPU, put it in the SLB right away */ if (pm == kernel_pmap && pmap_bootstrapped) { /* slbie not required */ __asm __volatile ("slbmte %0, %1" :: - "r"(kernel_pmap->pm_slb[to_spill].slbv), - "r"(kernel_pmap->pm_slb[to_spill].slbe)); + "r"(slbcache[to_spill].slbv), + "r"(slbcache[to_spill].slbe)); + } +} + +int +vsid_to_esid(pmap_t pm, uint64_t vsid, uint64_t *esid) +{ + uint64_t slbv; + struct slbcontainer *entry; + +#ifdef INVARIANTS + if (pm == kernel_pmap) + panic("vsid_to_esid only works on user pmaps"); + + PMAP_LOCK_ASSERT(pm, MA_OWNED); +#endif + + slbv = vsid << SLBV_VSID_SHIFT; + + SPLAY_FOREACH(entry, slb_tree, &pm->pm_slbtree) { + if (slbv == entry->slb.slbv) { + *esid = entry->slb.slbe >> SLBE_ESID_SHIFT; + return (0); + } + } + + return (-1); +} + +void +free_vsids(pmap_t pm) +{ + struct slbcontainer *entry; + + while (!SPLAY_EMPTY(&pm->pm_slbtree)) { + entry = SPLAY_MIN(slb_tree, &pm->pm_slbtree); + + SPLAY_REMOVE(slb_tree, &pm->pm_slbtree, entry); + + moea64_release_vsid(entry->slb.slbv >> SLBV_VSID_SHIFT); + uma_zfree(slb_zone, entry); } } +static int +slb_compare(struct slbcontainer *a, struct slbcontainer *b) +{ + if (a->slb.slbe == b->slb.slbe) + return (0); + else if (a->slb.slbe < b->slb.slbe) + return (-1); + else + return (1); +} + +static void +slb_zone_init(void *dummy) +{ + + slb_zone = uma_zcreate("SLB segment", sizeof(struct slbcontainer), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); + slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb), + NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM); +} + +struct slb * +slb_alloc_user_cache(void) +{ + struct slb *tmp; + tmp = uma_zalloc(slb_cache_zone, M_NOWAIT | M_ZERO); + bzero(tmp,64*sizeof(struct slb)); + return (tmp); +} + +void +slb_free_user_cache(struct slb *slb) +{ + uma_zfree(slb_cache_zone, slb); +} Modified: projects/ppc64/sys/powerpc/aim/trap.c ============================================================================== --- projects/ppc64/sys/powerpc/aim/trap.c Fri Mar 19 16:09:57 2010 (r205336) +++ projects/ppc64/sys/powerpc/aim/trap.c Fri Mar 19 16:15:11 2010 (r205337) @@ -88,8 +88,7 @@ static int handle_onfault(struct trapfra static void syscall(struct trapframe *frame); #ifdef __powerpc64__ -static void handle_slb_spill(pmap_t pm, vm_offset_t addr); -static uint64_t slb_esid_lookup(pmap_t pm, uint64_t vsid); +static int handle_slb_spill(pmap_t pm, vm_offset_t addr); #endif int setfault(faultbuf); /* defined in locore.S */ @@ -187,9 +186,10 @@ trap(struct trapframe *frame) #ifdef __powerpc64__ case EXC_ISE: case EXC_DSE: - handle_slb_spill(&p->p_vmspace->vm_pmap, + if (handle_slb_spill(&p->p_vmspace->vm_pmap, (type == EXC_ISE) ? frame->srr0 : - frame->cpu.aim.dar); + frame->cpu.aim.dar) != 0) + sig = SIGSEGV; break; #endif case EXC_DSI: @@ -251,9 +251,10 @@ trap(struct trapframe *frame) #ifdef __powerpc64__ case EXC_ISE: case EXC_DSE: - handle_slb_spill(kernel_pmap, + if (handle_slb_spill(kernel_pmap, (type == EXC_ISE) ? frame->srr0 : - frame->cpu.aim.dar); + frame->cpu.aim.dar) != 0) + panic("Fault handling kernel SLB miss"); return; #endif case EXC_MCHK: @@ -503,37 +504,40 @@ syscall(struct trapframe *frame) } #ifdef __powerpc64__ -static uint64_t -slb_esid_lookup(pmap_t pm, uint64_t vsid) +static int +handle_slb_spill(pmap_t pm, vm_offset_t addr) { - uint64_t esid; - int i; + struct slb slb_entry; + int error, i; - vsid <<= SLBV_VSID_SHIFT; + if (pm == kernel_pmap) { + error = va_to_slb_entry(pm, addr, &slb_entry); + if (error) + return (error); - for (i = 0; i < sizeof(pm->pm_slb)/sizeof(pm->pm_slb[0]); i++) { - if ((pm->pm_slb[i].slbv & SLBV_VSID_MASK) == vsid) { - esid = pm->pm_slb[i].slbe & SLBE_ESID_MASK; - esid >>= SLBE_ESID_SHIFT; - return (esid); - } + slb_insert(pm, PCPU_GET(slb), &slb_entry); + return (0); } - return (0); -} - -static void -handle_slb_spill(pmap_t pm, vm_offset_t addr) -{ - struct slb *slb_entry; - PMAP_LOCK(pm); - slb_entry = va_to_slb_entry(pm, addr); - if (slb_entry == NULL) - (void)va_to_vsid(pm, addr); - else - slb_insert(pm, slb_entry, 0 /* Don't prefer empty */); + error = va_to_slb_entry(pm, addr, &slb_entry); + if (error != 0) + (void)allocate_vsid(pm, (uintptr_t)addr >> ADDR_SR_SHFT, 0); + else { + /* + * Check that another CPU has not already mapped this. + * XXX: Per-thread SLB caches would be better. + */ + for (i = 0; i < 64; i++) + if (pm->pm_slb[i].slbe == (slb_entry.slbe | i)) + break; + + if (i == 64) + slb_insert(pm, pm->pm_slb, &slb_entry); + } PMAP_UNLOCK(pm); + + return (0); } #endif @@ -568,15 +572,22 @@ trap_pfault(struct trapframe *frame, int if (p->p_vmspace == NULL) return (SIGSEGV); + map = &p->p_vmspace->vm_map; + #ifdef __powerpc64__ user_sr = 0; __asm ("slbmfev %0, %1" : "=r"(user_sr) : "r"(USER_SR)); - user_sr = (user_sr & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT; - user_sr = slb_esid_lookup(&p->p_vmspace->vm_pmap, user_sr); + PMAP_LOCK(&p->p_vmspace->vm_pmap); + user_sr >>= SLBV_VSID_SHIFT; + rv = vsid_to_esid(&p->p_vmspace->vm_pmap, user_sr, + &user_sr); + PMAP_UNLOCK(&p->p_vmspace->vm_pmap); + if (rv != 0) + return (SIGSEGV); #else __asm ("mfsr %0, %1" : "=r"(user_sr) @@ -584,7 +595,6 @@ trap_pfault(struct trapframe *frame, int #endif eva &= ADDR_PIDX | ADDR_POFF; eva |= user_sr << ADDR_SR_SHFT; - map = &p->p_vmspace->vm_map; } else { map = kernel_map; } Modified: projects/ppc64/sys/powerpc/aim/trap_subr64.S ============================================================================== --- projects/ppc64/sys/powerpc/aim/trap_subr64.S Fri Mar 19 16:09:57 2010 (r205336) +++ projects/ppc64/sys/powerpc/aim/trap_subr64.S Fri Mar 19 16:15:11 2010 (r205337) @@ -46,7 +46,7 @@ /* * Restore SRs for a pmap * - * Requires that r28-r31 be scratch, with r28 initialized to the pmap + * Requires that r28-r31 be scratch, with r28 initialized to the SLB cache */ restoresrs: @@ -54,14 +54,15 @@ restoresrs: slbia slbmfee %r31,%r29 + clrrdi %r31,%r31,28 slbie %r31 instslb: - ld %r31, PM_SLB+8(%r28); /* Load SLBE */ + ld %r31, 8(%r28); /* Load SLBE */ cmpli 0, %r31, 0; /* If SLBE is not valid, get the next */ beq nslb; - ld %r30, PM_SLB(%r28) /* Load SLBV */ + ld %r30, 0(%r28) /* Load SLBV */ slbmte %r30, %r31; /* Install SLB entry */ nslb: @@ -77,15 +78,15 @@ nslb: */ #define RESTORE_USER_SRS() \ GET_CPUINFO(%r28); \ - ld %r28,PC_CURPMAP(%r28); \ + ld %r28,PC_USERSLB(%r28); \ bl restoresrs; /* * Kernel SRs are loaded directly from kernel_pmap_ */ #define RESTORE_KERN_SRS() \ - lis %r28,CNAME(kernel_pmap_store)@ha; \ - addi %r28,%r28,CNAME(kernel_pmap_store)@l; \ + GET_CPUINFO(%r28); \ + addi %r28,%r28,PC_KERNSLB; \ bl restoresrs; /* Modified: projects/ppc64/sys/powerpc/include/pcpu.h ============================================================================== --- projects/ppc64/sys/powerpc/include/pcpu.h Fri Mar 19 16:09:57 2010 (r205336) +++ projects/ppc64/sys/powerpc/include/pcpu.h Fri Mar 19 16:15:11 2010 (r205337) @@ -31,6 +31,7 @@ #define _MACHINE_PCPU_H_ #include <machine/cpufunc.h> +#include <machine/slb.h> #include <machine/tlb.h> struct pmap; @@ -50,7 +51,17 @@ struct pmap; register_t pc_disisave[CPUSAVE_LEN]; \ register_t pc_dbsave[CPUSAVE_LEN]; -#define PCPU_MD_AIM_FIELDS +#define PCPU_MD_AIM32_FIELDS + +#define PCPU_MD_AIM64_FIELDS \ + struct slb pc_slb[64]; \ + struct slb *pc_userslb; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201003191615.o2JGFBxk000295>