Date: Mon, 6 Mar 2006 08:15:01 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 92840 for review Message-ID: <200603060815.k268F1Ph034193@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=92840 Change 92840 by kmacy@kmacy_storage:sun4v_work on 2006/03/06 08:13:57 add support for data protection faults and remove any kernel dependency on TSBs for 8K pages Affected files ... .. //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#7 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/errata.h#1 add .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#18 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#17 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#7 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#4 edit Differences ... ==== //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#7 (text+ko) ==== @@ -180,8 +180,12 @@ ASSYM(TV_SIZE_BITS, TV_SIZE_BITS); #else ASSYM(VTD_REF, VTD_REF); -ASSYM(TTARGET_VA_MASK, TTARGET_VA_MASK); +ASSYM(VTD_W, VTD_W); +ASSYM(VTD_SW_W, VTD_SW_W); ASSYM(TTARGET_VA_BITS, TTARGET_VA_BITS); +ASSYM(TTARGET_VA_SHIFT, TTARGET_VA_SHIFT); +ASSYM(TTARGET_CTX_SHIFT, TTARGET_CTX_SHIFT); + ASSYM(THE_SHIFT, THE_SHIFT); #endif ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#18 (text+ko) ==== @@ -43,9 +43,11 @@ #include <machine/tstate.h> #include <machine/wstate.h> #include <machine/hypervisorvar.h> +#include <machine/errata.h> #include "assym.s" +#define PMAP_DEBUG #if 1 #define SPILL_FILL_MAGIC_TRAP_ON nop #define SPILL_FILL_MAGIC_TRAP_OFF nop @@ -258,8 +260,10 @@ .endm .macro insn_miss - MAGIC_TRAP_ON - illtrap + GET_MMFSA_SCRATCH(%g1) ! insn 1 + GET_HASH_SCRATCH(%g2) ! insn 2,3 + ba,pt %xcc, tsb_miss + mov VTD_REF, %g3 .align 32 .endm @@ -272,20 +276,25 @@ .macro data_miss GET_MMFSA_SCRATCH(%g1) ! insn 1 GET_HASH_SCRATCH(%g2) ! insn 2,3 - GET_TSB_SCRATCH(%g3) ! insn 4,5 + add %g1, MMFSA_D_, %g1 ! set fsa to data ba,pt %xcc, tsb_miss - add %g1, MMFSA_D_, %g1 ! set fsa to data + mov VTD_REF, %g3 .align 32 .endm .macro data_prot - MAGIC_TRAP_ON - illtrap + GET_MMFSA_SCRATCH(%g1) ! insn 1 + GET_HASH_SCRATCH(%g2) ! insn 2,3 + add %g1, MMFSA_D_, %g1 ! set fsa to data + ba,pt %xcc, tsb_miss + mov VTD_W, %g3 .align 32 .endm .macro tl0_align MAGIC_TRAP_ON + MAGIC_TRAP_ON + MAGIC_TRAP_ON illtrap .align 32 .endm @@ -641,7 +650,7 @@ tl0_reserved 18 ! 0x50-0x61 tl0_watch_virt_62: tl0_gen T_VA_WATCHPOINT ! 0x62 - tl0_reserved 10 ! 0x63-0x6c + tl0_reserved 9 ! 0x63-0x6b tl0_data_prot_6c: data_prot ! 0x6c tl0_reserved 8 ! 0x6d-0x75 @@ -1016,98 +1025,136 @@ END(tl0_intr) +! The HV documentation is wrong +! mappings are not already flushed befor taking a +! data protection trap +! IN: +! %g2 == hash base +! %g3 == flags +! %g5 == fault addr +! %g6 == context +! %g1,%g4,%g7 temps +! OUT: +! %g6 <- TAG +ontario_unmap_addr_errata_begin: + ! do the unmap call + mov %o0, %g1 + mov %o1, %g4 + mov %o2, %g7 + mov %g5, %o0 + mov %g6, %o1 + mov MAP_ITLB|MAP_DTLB, %o2 + ta MMU_UNMAP_ADDR + mov %g1, %o0 + mov %g4, %o1 + mov %g7, %o2 + srlx %g5, TTARGET_VA_SHIFT, %g4 + sllx %g6, TTARGET_CTX_SHIFT, %g6 + ba,pt %xcc, tsb_miss_fault_handler + or %g6, %g4, %g6 +ontario_unmap_addr_errata_end: + + + ! %g1==mmfsa (RA) ! %g2==hash base (VA) -! %g3==TSB (RA) +! %g3==TTE flags ! internal usage: ! %g1==absolute index +! %g2==pointer to hash entry +! %g3==flag bits, TSB (RA) ! %g4==fault type,entry tag -! %g5==fault address,entry data -! %g6==hash size,tag, temp +! %g5==tag +! %g6==hash size, temp ! %g7 temp ENTRY(tsb_miss) ldda [%g0 + %g1]ASI_LDTD_REAL, %g4 - ! %g4 == fault type %g5 == fault address - ! ignore context for now + addx %g1, 0x10, %g1 + ldda [%g0 + %g1]ASI_LDTD_REAL, %g6 + MAGIC_TRAP_ON +/* these two instructions will be patched + * at some point + */ +#ifdef ONTARIO_UNMAP_ERRATA + cmp %g3, VTD_W + be,pn %xcc, ontario_unmap_addr_errata_begin +#endif + srlx %g5, TTARGET_VA_SHIFT, %g1 + sllx %g6, TTARGET_CTX_SHIFT, %g6 + or %g6, %g1, %g6 ! %g6 == search tag + + +tsb_miss_fault_handler: + + ! %g4 == fault type %g5 == fault addr %g6 == tag ! XXX only handle normal miss for now - mov 1, %g7 +#ifdef PMAP_DEBUG + cmp %g6, %g0 ! NULL ptr deref in kernel + bne,pt %xcc, 4f + nop +tsb_miss_null_deref: + illtrap ! give up +4: +#endif + mov 1, %g7 sllx %g7, PAGE_SHIFT, %g7 sub %g7, 1, %g7 ! %g7==PAGE_MASK - MAGIC_TRAP_ON - and %g2, %g7, %g6 ! size stored in lower 13 bits + and %g2, %g7, %g4 ! size stored in lower 13 bits andn %g2, %g7, %g2 ! actual VA of hash ! XXX only handle 8k page miss ! calculate hash index srlx %g5, PAGE_SHIFT, %g1 ! absolute hash index - sllx %g6, (PAGE_SHIFT - THE_SHIFT), %g6 ! size of hash in THEs - sub %g6, 1, %g6 ! THE_MASK - and %g1, %g6, %g6 ! masked hash index - sllx %g6, THE_SHIFT, %g6 ! masked hash offset - srlx %g5, PAGE_SHIFT_4M, %g7 ! VA tag + sllx %g4, (PAGE_SHIFT - THE_SHIFT), %g4 ! size of hash in THEs + sub %g4, 1, %g4 ! THE_MASK + and %g1, %g4, %g4 ! masked hash index + sllx %g4, THE_SHIFT, %g4 ! masked hash offset ! fetch hash entries - exit when we find what were looking for ! %g2==entry base - add %g2, %g6, %g2 ! base + offset == entry base + add %g2, %g4, %g2 ! base + offset == entry base ! entry 0 -tsb_miss_lookup_0: - mov 1, %g6 - sllx %g6, TTARGET_VA_BITS, %g6 - subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK - + ! %g1 == abs index %g2 == THE pointer %g3 == flags + ! %g4 <- tag %g5 <- data + ! %g6 == search tag %g7 == PAGE_MASK +tsb_miss_lookup_0: ldda [%g2 + %g0]ASI_LDTD_N, %g4 - and %g4, %g6, %g6 ! mask off context bits - cmp %g6, %g0 ! entry tag == 0 + cmp %g4, %g0 ! entry tag == 0 ? be,pn %xcc, 1f nop - cmp %g6, %g7 ! entry tag == VA tag? + cmp %g4, %g6 ! entry tag == VA tag? be,pn %xcc, 2f nop ! entry 1 tsb_miss_lookup_1: - mov 1, %g6 - sllx %g6, TTARGET_VA_BITS, %g6 - subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK - - add %g2, 16, %g2 + add %g2, 16, %g2 ! next THF ldda [%g2 + %g0]ASI_LDTD_N, %g4 - and %g4, %g6, %g6 ! mask off context bits - cmp %g6, %g0 ! entry tag == 0 + cmp %g4, %g0 ! entry tag == 0 ? be,pn %xcc, 1f nop - cmp %g6, %g7 ! entry tag == VA tag? + cmp %g4, %g6 ! entry tag == search tag? be,pn %xcc, 2f nop ! entry 2 tsb_miss_lookup_2: - mov 1, %g6 - sllx %g6, TTARGET_VA_BITS, %g6 - subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK - - add %g2, 16, %g2 + add %g2, 16, %g2 ! next THF ldda [%g2 + %g0]ASI_LDTD_N, %g4 - and %g4, %g6, %g6 ! mask off context bits - cmp %g6, %g0 ! entry tag == 0 + cmp %g4, %g0 ! entry tag == 0 ? be,pn %xcc, 1f nop - cmp %g6, %g7 ! entry tag == VA tag? + cmp %g4, %g6 ! entry tag == search tag? be,pn %xcc, 2f nop ! entry 3 tsb_miss_lookup_3: - mov 1, %g6 - sllx %g6, TTARGET_VA_BITS, %g6 - subx %g6, 1, %g6 ! %g6 == TTARGET_VA_MASK - - add %g2, 16, %g2 + add %g2, 16, %g2 ! next THF ldda [%g2 + %g0]ASI_LDTD_N, %g4 - and %g4, %g6, %g6 ! mask off context bits - cmp %g6, %g0 ! entry tag == 0 + cmp %g4, %g0 ! entry tag == 0 ? be,pn %xcc, 1f nop - cmp %g6, %g7 ! entry tag == VA tag? + cmp %g4, %g6 ! entry tag == search tag? be,pn %xcc, 2f nop tsb_miss_not_found: @@ -1119,32 +1166,57 @@ tsb_miss_found: 2: !found - ! set referenced bit unconditionally for now - or %g5, VTD_REF, %g5 - stx %g5, [%g2 + 8] ! set ref bit + ! %g1 == abs index %g2 == THE pointer %g3 == flags + ! %g4 == tag %g5 == data %g7 == PAGE_MASK + ! %g3 <- TSB RA %g6 <- TSB size, TTE RA + + ! XXX set referenced/modified bit unconditionally for now XXX + andcc %g5, %g3, %g0 ! already set + bnz,pt %xcc, 5f + nop + andcc %g3, VTD_REF, %g0 ! TSB miss + bnz,pt %xcc, 6f + or %g5, %g3, %g5 ! add ref/mod bit unconditionally + andcc %g5, VTD_SW_W, %g0 ! write enabled? + bz,pn %xcc, prot_fault_trap ! write to read only page + or %g5, %g3, %g5 ! add ref/mod bit unconditionally +6: + stx %g5, [%g2 + 8] ! set ref/mod bit +5: + GET_TSB_SCRATCH(%g3) ! %g3 == TSB (RA) + - mov 1, %g7 - sllx %g7, PAGE_SHIFT, %g7 - sub %g7, 1, %g7 ! %g7==PAGE_MASK - - and %g3, %g7, %g6 ! size of TSB in pages + and %g3, %g7, %g6 ! size of TSB in pages andn %g3, %g7, %g3 ! TSB real address sllx %g6, (PAGE_SHIFT - TTE_SHIFT), %g6 ! nttes subx %g6, 1, %g6 ! TSB_MASK - and %g6, %g1, %g6 ! masked index + and %g6, %g1, %g6 ! masked index sllx %g6, TTE_SHIFT, %g6 ! masked byte offset - add %g6, %g3, %g6 ! TTE RA - mov 8, %g7 + add %g6, %g3, %g6 ! TTE RA + mov 8, %g7 +#ifdef PMAP_DEBUG + ldda [%g6]ASI_LDTD_REAL, %g2 + cmp %g3, %g5 + bne,pt %xcc, 3f + nop + illtrap ! die if all we're doing + ! is storing same data +3: +#endif stxa %g4, [%g6]ASI_REAL ! store tag stxa %g5, [%g6 + %g7]ASI_REAL ! store data MAGIC_TRAP_OFF + retry END(tsb_miss) - - - +/* + * Write to read-only page + */ +ENTRY(prot_fault_trap) + illtrap +END(prot_fault_trap) /* * Freshly forked processes come here when switched to for the first time. * The arguments to fork_exit() have been setup in the locals, we must move ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#17 (text+ko) ==== @@ -315,7 +315,6 @@ panic("pmap_bootstrap_alloc"); } - /* * Activate a user pmap. The pmap must be activated before its address space * can be accessed in any way. @@ -603,12 +602,19 @@ */ pm->pm_hash = tte_hash_kernel_create(kernel_hash, PAGE_SIZE_4M); tte_hash_set_scratchpad(pm->pm_hash); - - /* - * XXX - We should read the kernel mappings into the hash table - * - */ + for (i = 0; i < sz; i++) { + if (translations[i].om_start < VM_MIN_PROM_ADDRESS || + translations[i].om_start > VM_MAX_PROM_ADDRESS) + continue; + if (translations[i].om_size == PAGE_SIZE_4M) + continue; + for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) { + va = translations[i].om_start + off; + pa = TTE_GET_PA(translations[i].om_tte) + off; + tte_hash_insert(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K); + } + } } @@ -691,7 +697,7 @@ tte_data = pa = VM_PAGE_TO_PHYS(m); otte = tte_hash_lookup(pmap->pm_hash, va); - otte_data = *otte; + otte_data = otte ? *otte : 0; opa = TTE_GET_PA(otte_data); /* * Mapping has not changed, must be protection or wiring change. @@ -780,8 +786,9 @@ } if (invlva) pmap_invalidate_page(pmap, va); - } else + } else { tte_hash_insert(pmap->pm_hash, va, tte_data); + } } @@ -1000,17 +1007,16 @@ vm_paddr_t pmap_kextract(vm_offset_t va) { - uint64_t tte_data; + uint64_t *tte, tte_data; vm_paddr_t pa; pa = 0; -#if 0 - printf("tte_data=%lx TTE_GET_PA(tte_data)=%lx (va & TTE_GET_PAGE_MASK(tte_data))=%lx\n", - tsb_lookup_tte(va, 0), TTE_GET_PA(tte_data), (va & TTE_GET_PAGE_MASK(tte_data))); -#endif if ((tte_data = tsb_lookup_tte(va, 0)) != 0) pa = TTE_GET_PA(tte_data) | (va & TTE_GET_PAGE_MASK(tte_data)); + if ((pa == 0) && (tte = tte_hash_lookup(kernel_pmap->pm_hash, va)) != NULL) + pa = TTE_GET_PA(*tte) | (va & TTE_GET_PAGE_MASK(*tte)); + return pa; } @@ -1290,19 +1296,14 @@ */ if (pmap->pm_stats.resident_count == 0) return; - + + invlva = 0; vm_page_lock_queues(); sched_pin(); PMAP_LOCK(pmap); - if (pmap->pm_context != 0) { - invlva = 0; - for (tva = start; tva < end; tva += PAGE_SIZE) { - tinvlva = tte_hash_delete(pmap->pm_hash, tva); - invlva = tinvlva ? tinvlva : invlva; - } - } else { - tsb_clear_range(&pmap->pm_tsb, start, end); - invlva = 1; + for (tva = start; tva < end; tva += PAGE_SIZE) { + tinvlva = tte_hash_delete(pmap->pm_hash, tva); + invlva = tinvlva ? tinvlva : invlva; } sched_unpin(); vm_page_unlock_queues(); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#7 (text+ko) ==== @@ -55,7 +55,6 @@ * void cpu_switch(struct thread *old, struct thread *new) */ ENTRY(cpu_switch) - MAGIC_TRAP_ON GET_PCB(PCB_REG) save %sp, -CCFSZ, %sp mov %i1, %i0 ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#4 (text+ko) ==== @@ -129,17 +129,21 @@ th->th_hashtable = (tte_hash_entry_t)kmem_alloc_nofault(kernel_map, PAGE_SIZE*HASH_SIZE); - for (i = 0; i < HASH_SIZE; i++) { + printf("th->th_hashtable=%p ", th->th_hashtable); + for (i = 0; i < HASH_SIZE;) { m = vm_page_alloc(NULL, color++, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); + printf("PHYS(m)=0x%010lx ", VM_PAGE_TO_PHYS(m)); if (m == NULL) VM_WAIT; else { hash_pages[i++] = m; } } + printf("entered\n"); pmap_qenter((vm_offset_t)th->th_hashtable, hash_pages, HASH_SIZE); + for (i = 0; i < HASH_SIZE; i++) { if ((hash_pages[i]->flags & PG_ZERO) == 0) pmap_zero_page(hash_pages[i]); @@ -188,7 +192,7 @@ KASSERT(tte_hash_lookup(th, va) != 0, ("attempting to delete non-existent entry")); #endif for (i = 0; i <= 3; i++) - if ((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & PAGE_MASK_4M)) + if ((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M)) break; vaindex = i; @@ -215,18 +219,16 @@ uint64_t hash_shift, hash_index, tte_tag; tte_hash_field_t fields; int i; + /* XXX - only handle 8K pages for now */ - hash_shift = PAGE_SHIFT; hash_index = (va >> hash_shift) & HASH_MASK(th); - fields = (th->th_hashtable[hash_index].the_fields); tte_tag = (((uint64_t)th->th_context << TTARGET_CTX_SHIFT)|(va >> TTARGET_VA_SHIFT)); for (i = 0; i <= 3; i++) { if ((fields[i].tte.tag == 0) || (fields[i].tte.tag == tte_tag)) { fields[i].tte.data = tte_data; fields[i].tte.tag = tte_tag; - printf("data: 0x%016lx tag: 0x%016lx\n", fields[i].tte.data, fields[i].tte.tag); goto done; } @@ -250,7 +252,7 @@ fields = (th->th_hashtable[hash_index].the_fields); for (i = 0; i <= 3; i++) { - if ((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & PAGE_MASK_4M)) + if ((fields[i].tte.tag << TTARGET_VA_SHIFT) == (va & ~PAGE_MASK_4M)) return &(fields[i].tte.data); } /*
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200603060815.k268F1Ph034193>