From owner-p4-projects@FreeBSD.ORG Sun Mar 5 10:18:22 2006 Return-Path: X-Original-To: p4-projects@freebsd.org Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id CFB6016A423; Sun, 5 Mar 2006 10:18:21 +0000 (GMT) X-Original-To: perforce@freebsd.org Delivered-To: perforce@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id A58C016A420 for ; Sun, 5 Mar 2006 10:18:21 +0000 (GMT) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [216.136.204.115]) by mx1.FreeBSD.org (Postfix) with ESMTP id 57A5043D46 for ; Sun, 5 Mar 2006 10:18:21 +0000 (GMT) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.13.1/8.13.1) with ESMTP id k25AILNg040221 for ; Sun, 5 Mar 2006 10:18:21 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.13.1/8.13.1/Submit) id k25AIL6Y040203 for perforce@freebsd.org; Sun, 5 Mar 2006 10:18:21 GMT (envelope-from kmacy@freebsd.org) Date: Sun, 5 Mar 2006 10:18:21 GMT Message-Id: <200603051018.k25AIL6Y040203@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 92773 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 05 Mar 2006 10:18:22 -0000 http://perforce.freebsd.org/chv.cgi?CH=92773 Change 92773 by kmacy@kmacy_storage:sun4v_work on 2006/03/05 10:18:20 add basic TSB miss handling support Affected files ... .. //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#6 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/param.h#5 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/sun4v_cpufunc.h#2 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#8 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#3 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#17 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#16 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#6 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#7 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#3 edit Differences ... ==== //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#6 (text+ko) ==== @@ -71,6 +71,11 @@ #include #include #include +#ifdef SUN4V +#include +#include +#endif + ASSYM(KERNBASE, KERNBASE); ASSYM(VM_MIN_PROM_ADDRESS, VM_MIN_PROM_ADDRESS); @@ -100,13 +105,16 @@ ASSYM(TLB_DEMAP_CONTEXT, TLB_DEMAP_CONTEXT); ASSYM(TLB_DEMAP_PAGE, TLB_DEMAP_PAGE); +#ifndef SUN4V ASSYM(TSB_BUCKET_MASK, TSB_BUCKET_MASK); ASSYM(TSB_BUCKET_SHIFT, TSB_BUCKET_SHIFT); - +#endif ASSYM(INT_SHIFT, INT_SHIFT); ASSYM(PTR_SHIFT, PTR_SHIFT); ASSYM(PAGE_SHIFT, PAGE_SHIFT); +ASSYM(PAGE_MASK, PAGE_MASK); +ASSYM(PAGE_MASK_4M, PAGE_MASK_4M); ASSYM(PAGE_SHIFT_8K, PAGE_SHIFT_8K); ASSYM(PAGE_SHIFT_4M, PAGE_SHIFT_4M); ASSYM(PAGE_SIZE, PAGE_SIZE); @@ -170,6 +178,11 @@ ASSYM(TS_MAX, TS_MAX); ASSYM(TLB_DIRECT_TO_TTE_MASK, TLB_DIRECT_TO_TTE_MASK); ASSYM(TV_SIZE_BITS, TV_SIZE_BITS); +#else +ASSYM(VTD_REF, VTD_REF); +ASSYM(TTARGET_VA_MASK, TTARGET_VA_MASK); +ASSYM(TTARGET_VA_BITS, TTARGET_VA_BITS); +ASSYM(THE_SHIFT, THE_SHIFT); #endif ASSYM(V_INTR, offsetof(struct vmmeter, v_intr)); @@ -198,6 +211,11 @@ ASSYM(PC_RQ_SIZE, offsetof(struct pcpu, pc_rq_size)); ASSYM(PC_NRQ_BASE, offsetof(struct pcpu, pc_nrq_ra)); ASSYM(PC_NRQ_SIZE, offsetof(struct pcpu, pc_nrq_size)); + +ASSYM(PC_KWBUF_FULL, offsetof(struct pcpu, pc_kwbuf_full)); +ASSYM(PC_KWBUF_SP, offsetof(struct pcpu, pc_kwbuf_sp)); +ASSYM(PC_KWBUF, offsetof(struct pcpu, pc_kwbuf)); + #else ASSYM(PC_PMAP, offsetof(struct pcpu, pc_pmap)); ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb)); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/param.h#5 (text+ko) ==== @@ -120,6 +120,7 @@ /* * Mach derived conversion macros */ +#ifndef LOCORE #define round_page(x) (((unsigned long)(x) + PAGE_MASK) & ~PAGE_MASK) #define trunc_page(x) ((unsigned long)(x) & ~PAGE_MASK) @@ -130,10 +131,8 @@ #define sparc64_ptob(x) ((unsigned long)(x) << PAGE_SHIFT) #define pgtok(x) ((unsigned long)(x) * (PAGE_SIZE / 1024)) +#endif /* LOCORE */ -#define NPGPTD 1 /* number of page table directory pages */ -#define NBPTD (NPGPTD << PAGE_SHIFT) /* number of bytes in a page table directory */ -#define NPDEPG (PAGE_SIZE/(sizeof (vm_offset_t))) #endif /* !_MACHINE_PARAM_H_ */ ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/sun4v_cpufunc.h#2 (text+ko) ==== @@ -32,7 +32,7 @@ #define _MACHINE_SUN4V_CPUFUNC_H_ #include void set_mmfsa_scratchpad(vm_paddr_t mmfsa); -void set_hash_scratchpad(void *hash); +void set_hash_scratchpad(vm_offset_t hash); void set_tsb_scratchpad(vm_paddr_t tsb); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#8 (text+ko) ==== @@ -38,4 +38,6 @@ void tsb_clear_range(struct hv_tsb_info *tsb, vm_offset_t sva, vm_offset_t eva); +void tsb_set_scratchpad(struct hv_tsb_info *tsb); + #endif /* !_MACHINE_TSB_H_ */ ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#3 (text+ko) ==== @@ -1,6 +1,7 @@ #ifndef _MACHINE_TTE_HASH_H_ #define _MACHINE_TTE_HASH_H_ +#define THE_SHIFT 6 /* size of hash entry is 64-bytes */ struct tte_hash; typedef struct tte_hash *tte_hash_t; ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#17 (text+ko) ==== @@ -42,6 +42,7 @@ #include #include #include +#include #include "assym.s" @@ -228,7 +229,7 @@ wrpr %g1, WSTATE_NESTED, %wstate save %sp, -(CCFSZ + TF_SIZEOF), %sp #endif - nop + illtrap .endm .macro tl1_setup type @@ -252,32 +253,40 @@ .macro insn_excptn MAGIC_TRAP_ON + illtrap .align 32 .endm .macro insn_miss MAGIC_TRAP_ON + illtrap .align 32 .endm .macro data_excptn MAGIC_TRAP_ON + illtrap .align 32 .endm .macro data_miss - MAGIC_TRAP_ON - + GET_MMFSA_SCRATCH(%g1) ! insn 1 + GET_HASH_SCRATCH(%g2) ! insn 2,3 + GET_TSB_SCRATCH(%g3) ! insn 4,5 + ba,pt %xcc, tsb_miss + add %g1, MMFSA_D_, %g1 ! set fsa to data .align 32 .endm .macro data_prot MAGIC_TRAP_ON + illtrap .align 32 .endm .macro tl0_align MAGIC_TRAP_ON + illtrap .align 32 .endm @@ -1007,6 +1016,134 @@ END(tl0_intr) +! %g1==mmfsa (RA) +! %g2==hash base (VA) +! %g3==TSB (RA) +! internal usage: +! %g1==absolute index +! %g4==fault type,entry tag +! %g5==fault address,entry data +! %g6==hash size,tag, temp +! %g7 temp +ENTRY(tsb_miss) + ldda [%g0 + %g1]ASI_LDTD_REAL, %g4 + ! %g4 == fault type %g5 == fault address + ! ignore context for now + ! XXX only handle normal miss for now + mov 1, %g7 + sllx %g7, PAGE_SHIFT, %g7 + sub %g7, 1, %g7 ! %g7==PAGE_MASK + + MAGIC_TRAP_ON + and %g2, %g7, %g6 ! size stored in lower 13 bits + andn %g2, %g7, %g2 ! actual VA of hash + + ! XXX only handle 8k page miss + ! calculate hash index + srlx %g5, PAGE_SHIFT, %g1 ! absolute hash index + sllx %g6, (PAGE_SHIFT - THE_SHIFT), %g6 ! size of hash in THEs + sub %g6, 1, %g6 ! THE_MASK + and %g1, %g6, %g6 ! masked hash index + sllx %g6, THE_SHIFT, %g6 ! masked hash offset + srlx %g5, PAGE_SHIFT_4M, %g7 ! VA tag + ! fetch hash entries - exit when we find what were looking for + + ! %g2==entry base + add %g2, %g6, %g2 ! base + offset == entry base + + ! entry 0 +tsb_miss_lookup_0: + mov 1, %g6 + sllx %g6, TTARGET_VA_BITS, %g6 + subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK + + ldda [%g2 + %g0]ASI_LDTD_N, %g4 + and %g4, %g6, %g6 ! mask off context bits + cmp %g6, %g0 ! entry tag == 0 + be,pn %xcc, 1f + nop + cmp %g6, %g7 ! entry tag == VA tag? + be,pn %xcc, 2f + nop + ! entry 1 +tsb_miss_lookup_1: + mov 1, %g6 + sllx %g6, TTARGET_VA_BITS, %g6 + subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK + + add %g2, 16, %g2 + ldda [%g2 + %g0]ASI_LDTD_N, %g4 + and %g4, %g6, %g6 ! mask off context bits + cmp %g6, %g0 ! entry tag == 0 + be,pn %xcc, 1f + nop + cmp %g6, %g7 ! entry tag == VA tag? + be,pn %xcc, 2f + nop + ! entry 2 +tsb_miss_lookup_2: + mov 1, %g6 + sllx %g6, TTARGET_VA_BITS, %g6 + subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK + + add %g2, 16, %g2 + ldda [%g2 + %g0]ASI_LDTD_N, %g4 + and %g4, %g6, %g6 ! mask off context bits + cmp %g6, %g0 ! entry tag == 0 + be,pn %xcc, 1f + nop + cmp %g6, %g7 ! entry tag == VA tag? + be,pn %xcc, 2f + nop + ! entry 3 +tsb_miss_lookup_3: + mov 1, %g6 + sllx %g6, TTARGET_VA_BITS, %g6 + subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK + + add %g2, 16, %g2 + ldda [%g2 + %g0]ASI_LDTD_N, %g4 + and %g4, %g6, %g6 ! mask off context bits + cmp %g6, %g0 ! entry tag == 0 + be,pn %xcc, 1f + nop + cmp %g6, %g7 ! entry tag == VA tag? + be,pn %xcc, 2f + nop +tsb_miss_not_found: +1: ! not found + ! we need to jump to tl0_trap to drop us back down to tl0 + ! and take us to trap(...) to service the fault + ! skipping this step for the moment so we just do an illtrap + illtrap + +tsb_miss_found: +2: !found + ! set referenced bit unconditionally for now + or %g5, VTD_REF, %g5 + stx %g5, [%g2 + 8] ! set ref bit + + mov 1, %g7 + sllx %g7, PAGE_SHIFT, %g7 + sub %g7, 1, %g7 ! %g7==PAGE_MASK + + and %g3, %g7, %g6 ! size of TSB in pages + + andn %g3, %g7, %g3 ! TSB real address + sllx %g6, (PAGE_SHIFT - TTE_SHIFT), %g6 ! nttes + subx %g6, 1, %g6 ! TSB_MASK + and %g6, %g1, %g6 ! masked index + sllx %g6, TTE_SHIFT, %g6 ! masked byte offset + add %g6, %g3, %g6 ! TTE RA + mov 8, %g7 + stxa %g4, [%g6]ASI_REAL ! store tag + stxa %g5, [%g6 + %g7]ASI_REAL ! store data + MAGIC_TRAP_OFF + retry +END(tsb_miss) + + + /* * Freshly forked processes come here when switched to for the first time. ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#16 (text+ko) ==== @@ -355,7 +355,7 @@ { struct pmap *pm; vm_offset_t off, va, kernel_hash; - vm_paddr_t pa; + vm_paddr_t pa, kernel_hash_pa; vm_size_t physsz, virtsz; ihandle_t pmem, vmem; int i, sz, j; @@ -411,18 +411,26 @@ virtsz = roundup(physsz, PAGE_SIZE_4M << (PAGE_SHIFT - TTE_SHIFT)); vm_max_kernel_address = VM_MIN_KERNEL_ADDRESS + virtsz; - + /* + * Set the start and end of kva. The kernel is loaded at the first + * available 4 meg super page, so round up to the end of the page. + */ + virtual_avail = roundup2(ekva, PAGE_SIZE_4M); + virtual_end = vm_max_kernel_address; + kernel_vm_end = vm_max_kernel_address; /* * Allocate and map a 4MB page for the kernel hashtable * */ - pa = pmap_bootstrap_alloc(PAGE_SIZE_4M); - if (pa & PAGE_MASK_4M) - panic("pmap_bootstrap: hashtable unaligned\n"); + kernel_hash_pa = pmap_bootstrap_alloc(PAGE_SIZE_4M); + if (kernel_hash_pa & PAGE_MASK_4M) + panic("pmap_bootstrap: hashtable pa unaligned\n"); kernel_hash = virtual_avail; + if (kernel_hash & PAGE_MASK_4M) + panic("pmap_bootstrap: hashtable va unaligned\n"); virtual_avail += PAGE_SIZE_4M; - pmap_scrub_pages(pa, PAGE_SIZE_4M); + pmap_scrub_pages(kernel_hash_pa, PAGE_SIZE_4M); /* * Set up TSB descriptors for the hypervisor @@ -453,7 +461,7 @@ kernel_pmap->pm_tsb.hvtsb_rsvd = 0; kernel_pmap->pm_tsb.hvtsb_pa = pa; - set_tsb_scratchpad(pa); + tsb_set_scratchpad(&kernel_pmap->pm_tsb); /* * Initialize kernel TSB for 4M pages @@ -480,7 +488,7 @@ * */ tsb_set_tte(&kernel_td[TSB4M_INDEX], kernel_hash, - pa | TTE_KERNEL | VTD_4M, 0); + kernel_hash_pa | TTE_KERNEL | VTD_4M, 0); /* * allocate MMU fault status areas for all CPUS @@ -501,15 +509,6 @@ #endif /* - * Set the start and end of kva. The kernel is loaded at the first - * available 4 meg super page, so round up to the end of the page. - */ - virtual_avail = roundup2(ekva, PAGE_SIZE_4M); - virtual_end = vm_max_kernel_address; - kernel_vm_end = vm_max_kernel_address; - - - /* * Allocate a kernel stack with guard page for thread0 and map it into * the kernel tsb. */ @@ -603,6 +602,7 @@ * attempts to do updates until they're legal */ pm->pm_hash = tte_hash_kernel_create(kernel_hash, PAGE_SIZE_4M); + tte_hash_set_scratchpad(pm->pm_hash); /* * XXX - We should read the kernel mappings into the hash table @@ -989,7 +989,7 @@ void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { - tsb_set_tte(&kernel_td[TSB8K_INDEX], va, pa | TTE_KERNEL | VTD_8K, 0); + tte_hash_insert(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K); } /* @@ -1022,7 +1022,7 @@ tsb_get_tte(&kernel_td[TSB4M_INDEX], va) != 0) tsb_clear_tte(&kernel_td[TSB4M_INDEX], va); else - tsb_clear_tte(&kernel_td[TSB8K_INDEX], va); + tte_hash_delete(kernel_pmap->pm_hash, va); } static void ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#6 (text+ko) ==== @@ -39,7 +39,8 @@ #define PCB_REG %g6 - +#define MAGIC_TRAP_ON ta 0x77 +#define MAGIC_TRAP_OFF ta 0x78 /* * void cpu_throw(struct thread *old, struct thread *new) */ @@ -54,6 +55,7 @@ * void cpu_switch(struct thread *old, struct thread *new) */ ENTRY(cpu_switch) + MAGIC_TRAP_ON GET_PCB(PCB_REG) save %sp, -CCFSZ, %sp mov %i1, %i0 @@ -243,7 +245,7 @@ stxa %i4, [%i5] ASI_IMMU mov AA_DMMU_PCXR, %i5 stxa %i3, [%i5] ASI_DMMU - membar #Sync + membar #Sync /* Check the hypervisor spec */ #endif /* * Done. Return and load the new process's window from the stack. ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#7 (text+ko) ==== @@ -184,3 +184,10 @@ return tte_data; } +void +tsb_set_scratchpad(hv_tsb_info_t *tsb) +{ + uint64_t tsb_pages; + tsb_pages = tsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT); + set_tsb_scratchpad(tsb->hvtsb_pa | tsb_pages); +} ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#3 (text+ko) ==== @@ -26,7 +26,7 @@ #define HASH_SIZE 4 #define MAX_HASH_SIZE 16 -#define HASH_MASK(th) ((th->th_size << PAGE_SHIFT) - 1) +#define HASH_MASK(th) ((th->th_size << (PAGE_SHIFT - THE_SHIFT)) - 1) #define HASH_VALID 0x1 struct tte_hash_entry; @@ -106,6 +106,8 @@ th->th_size = (size >> PAGE_SHIFT); th->th_entries = 0; th->th_context = 0; + printf("setting kernel hashtable to %lx\n", va); + th->th_hashtable = (tte_hash_entry_t)va; return th; } @@ -217,15 +219,17 @@ hash_shift = PAGE_SHIFT; hash_index = (va >> hash_shift) & HASH_MASK(th); + fields = (th->th_hashtable[hash_index].the_fields); - tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)||(va >> TTARGET_VA_SHIFT)); - + tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT)); for (i = 0; i <= 3; i++) { - if ((fields[i].tte.data == 0) || (fields[i].tte.tag == tte_tag)) { + if ((fields[i].tte.tag == 0) || (fields[i].tte.tag == tte_tag)) { fields[i].tte.data = tte_data; fields[i].tte.tag = tte_tag; + printf("data: 0x%016lx tag: 0x%016lx\n", fields[i].tte.data, fields[i].tte.tag); goto done; - } + } + } panic("collision handling unimplemented - please re-consider"); @@ -259,6 +263,11 @@ void tte_hash_set_scratchpad(tte_hash_t th) { - set_hash_scratchpad(th->th_hashtable); + /* This will break if a hash table ever grows above 64MB + * 2^(13+13) + */ + printf("setting hash scratch to %lx\n", + ((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size)); + set_hash_scratchpad(((vm_offset_t)th->th_hashtable) | ((vm_offset_t)th->th_size)); }