From owner-p4-projects@FreeBSD.ORG Thu Mar 9 08:04:51 2006 Return-Path: X-Original-To: p4-projects@freebsd.org Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id A1E2416A423; Thu, 9 Mar 2006 08:04:50 +0000 (GMT) X-Original-To: perforce@freebsd.org Delivered-To: perforce@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id 44BAE16A420 for ; Thu, 9 Mar 2006 08:04:50 +0000 (GMT) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (repoman.freebsd.org [216.136.204.115]) by mx1.FreeBSD.org (Postfix) with ESMTP id 12FD043D53 for ; Thu, 9 Mar 2006 08:04:44 +0000 (GMT) (envelope-from kmacy@freebsd.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.13.1/8.13.1) with ESMTP id k2984hlp067407 for ; Thu, 9 Mar 2006 08:04:43 GMT (envelope-from kmacy@freebsd.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.13.1/8.13.1/Submit) id k2984h2J067404 for perforce@freebsd.org; Thu, 9 Mar 2006 08:04:43 GMT (envelope-from kmacy@freebsd.org) Date: Thu, 9 Mar 2006 08:04:43 GMT Message-Id: <200603090804.k2984h2J067404@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to kmacy@freebsd.org using -f From: Kip Macy To: Perforce Change Reviews Cc: Subject: PERFORCE change 93021 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 09 Mar 2006 08:04:51 -0000 http://perforce.freebsd.org/chv.cgi?CH=93021 Change 93021 by kmacy@kmacy_storage:sun4v_work on 2006/03/09 08:03:46 add missing bits to support running as user - user_rtt: return to user mod - cpu_switch: handle new pmap structures - handle both user and kernel misses in tsb miss handler still need to connect up tl0_trap to transition to trap for demand fault Affected files ... .. //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#8 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/asi.h#11 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/asmacros.h#6 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/mmu.h#2 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/pmap.h#7 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/sun4v_cpufunc.h#3 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#9 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#4 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#21 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/hcall.S#5 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/machdep.c#9 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#19 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/support.S#9 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#8 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tsb.c#8 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte.c#2 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/tte_hash.c#6 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/wbuf.S#2 edit Differences ... ==== //depot/projects/kmacy_sun4v/src/sys/sparc64/sparc64/genassym.c#8 (text+ko) ==== @@ -118,6 +118,7 @@ ASSYM(PAGE_SHIFT_8K, PAGE_SHIFT_8K); ASSYM(PAGE_SHIFT_4M, PAGE_SHIFT_4M); ASSYM(PAGE_SIZE, PAGE_SIZE); +ASSYM(PAGE_SIZE_4M, PAGE_SIZE_4M); ASSYM(CPU_CLKSYNC, CPU_CLKSYNC); ASSYM(CPU_INIT, CPU_INIT); @@ -182,11 +183,12 @@ ASSYM(VTD_REF, VTD_REF); ASSYM(VTD_W, VTD_W); ASSYM(VTD_SW_W, VTD_SW_W); -ASSYM(TTARGET_VA_BITS, TTARGET_VA_BITS); -ASSYM(TTARGET_VA_SHIFT, TTARGET_VA_SHIFT); -ASSYM(TTARGET_CTX_SHIFT, TTARGET_CTX_SHIFT); ASSYM(THE_SHIFT, THE_SHIFT); +ASSYM(HVTSB_PA, offsetof(struct hv_tsb_info, hvtsb_pa)); +ASSYM(PM_HASHSCRATCH, offsetof(struct pmap, pm_hashscratch)); +ASSYM(PM_TSBSCRATCH, offsetof(struct pmap, pm_tsbscratch)); +ASSYM(PM_TSB_RA, offsetof(struct pmap, pm_tsb_ra)); #endif ASSYM(V_INTR, offsetof(struct vmmeter, v_intr)); @@ -219,10 +221,9 @@ ASSYM(PC_KWBUF_FULL, offsetof(struct pcpu, pc_kwbuf_full)); ASSYM(PC_KWBUF_SP, offsetof(struct pcpu, pc_kwbuf_sp)); ASSYM(PC_KWBUF, offsetof(struct pcpu, pc_kwbuf)); - +ASSYM(PC_PMAP, offsetof(struct pcpu, pc_curpmap)); #else ASSYM(PC_PMAP, offsetof(struct pcpu, pc_pmap)); -ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb)); #endif @@ -287,7 +288,7 @@ ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); ASSYM(PM_CONTEXT, offsetof(struct pmap, pm_context)); - +ASSYM(PM_TSB, offsetof(struct pmap, pm_tsb)); ASSYM(SF_UC, offsetof(struct sigframe, sf_uc)); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/asi.h#11 (text+ko) ==== @@ -127,8 +127,10 @@ #define SCRATCH_REG_MMFSA ASI_SCRATCHPAD_0_REG #define SCRATCH_REG_PCPU ASI_SCRATCHPAD_1_REG -#define SCRATCH_REG_HASH ASI_SCRATCHPAD_2_REG -#define SCRATCH_REG_TSB ASI_SCRATCHPAD_3_REG +#define SCRATCH_REG_HASH_KERNEL ASI_SCRATCHPAD_2_REG +#define SCRATCH_REG_TSB_KERNEL ASI_SCRATCHPAD_3_REG +#define SCRATCH_REG_HASH_USER ASI_SCRATCHPAD_6_REG +#define SCRATCH_REG_TSB_USER ASI_SCRATCHPAD_7_REG #define MMU_CID_P 0x08 #define MMU_CID_S 0x10 ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/asmacros.h#6 (text+ko) ==== @@ -135,21 +135,33 @@ mov SCRATCH_REG_PCPU, PCPU_REG; \ ldxa [%g0 + PCPU_REG]ASI_SCRATCHPAD, PCPU_REG; -#define GET_HASH_SCRATCH(reg) \ - mov SCRATCH_REG_HASH, reg; \ +#define GET_HASH_SCRATCH_USER(reg) \ + mov SCRATCH_REG_HASH_USER, reg; \ ldxa [%g0 + reg]ASI_SCRATCHPAD, reg; -#define GET_TSB_SCRATCH(reg) \ - mov SCRATCH_REG_TSB, reg; \ +#define GET_HASH_SCRATCH_KERNEL(reg) \ + mov SCRATCH_REG_HASH_KERNEL, reg; \ + ldxa [%g0 + reg]ASI_SCRATCHPAD, reg; + +#define GET_TSB_SCRATCH_USER(reg) \ + mov SCRATCH_REG_TSB_USER, reg; \ ldxa [%g0 + reg]ASI_SCRATCHPAD, reg; -#define SET_HASH_SCRATCH(reg) stxa reg, [%g0 + SCRATCH_REG_HASH]ASI_SCRATCHPAD -#define SET_TSB_SCRATCH(reg) stxa reg, [%g0 + SCRATCH_REG_TSB]ASI_SCRATCHPAD +#define GET_TSB_SCRATCH_KERNEL(reg) \ + mov SCRATCH_REG_TSB_KERNEL, reg; \ + ldxa [%g0 + reg]ASI_SCRATCHPAD, reg; + +#define SET_SCRATCH(offsetreg, reg) stxa reg, [%g0 + offsetreg]ASI_SCRATCHPAD #define GET_PCB(reg) \ GET_PCPU_SCRATCH; \ ldx [PCPU_REG + PC_CURPCB], reg; +#define SET_MMU_CONTEXT(typereg, reg) stxa reg, [typereg]ASI_MMU_CONTEXTID +#define GET_MMU_CONTEXT(typereg, reg) ldxa [typereg]ASI_MMU_CONTEXTID, reg + + + #define SAVE_GLOBALS(TF) \ stx %g1, [TF + TF_G1]; \ stx %g2, [TF + TF_G2]; \ ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/mmu.h#2 (text+ko) ==== @@ -76,19 +76,6 @@ /* - * MMU TAG ACCESS register Layout - * - * +-------------------------+------------------+ - * | virtual address [63:13] | context [12:0] | - * +-------------------------+------------------+ - * 63 13 12 0 - */ -#define TAGACC_CTX_MASK 0x1FFF -#define TAGACC_SHIFT 13 -#define TAGACC_VADDR_MASK (~TAGACC_CTX_MASK) -#define TAGACC_CTX_LSHIFT (64 - TAGACC_SHIFT) - -/* * MMU PRIMARY/SECONDARY CONTEXT register */ #define CTXREG_CTX_MASK 0x1FFF @@ -103,6 +90,6 @@ * FLUSH_ADDR is used in the flush instruction to guarantee stores to mmu * registers complete. It is selected so it won't miss in the tlb. */ -#define FLUSH_ADDR (KERNELBASE + 2 * MMU_PAGESIZE4M) +#define FLUSH_ADDR (KERNBASE + 2 * PAGE_SIZE_4M) #endif /* _MACHINE_MMU_H_ */ ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/pmap.h#7 (text+ko) ==== @@ -59,12 +59,16 @@ struct pmap { + uint64_t pm_context; + uint64_t pm_hashscratch; + uint64_t pm_tsbscratch; + vm_paddr_t pm_tsb_ra; + struct mtx pm_mtx; struct tte_hash *pm_hash; TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ struct hv_tsb_info pm_tsb; cpumask_t pm_active; - uint16_t pm_context; struct pmap_statistics pm_stats; }; ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/sun4v_cpufunc.h#3 (text+ko) ==== @@ -32,8 +32,11 @@ #define _MACHINE_SUN4V_CPUFUNC_H_ #include void set_mmfsa_scratchpad(vm_paddr_t mmfsa); -void set_hash_scratchpad(vm_offset_t hash); -void set_tsb_scratchpad(vm_paddr_t tsb); + +void set_hash_user_scratchpad(uint64_t); +void set_tsb_user_scratchpad(uint64_t); +void set_hash_kernel_scratchpad(uint64_t); +void set_tsb_kernel_scratchpad(uint64_t); static __inline void * ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tsb.h#9 (text+ko) ==== @@ -24,6 +24,10 @@ struct hv_tsb_info; +vm_paddr_t tsb_init(struct hv_tsb_info *tsb); + +void tsb_deinit(struct hv_tsb_info *tsb); + void tsb_assert_invalid(struct hv_tsb_info *tsb, vm_offset_t va); void tsb_set_tte(struct hv_tsb_info *tsb, vm_offset_t va, tte_t tte_data, uint64_t ctx); @@ -38,6 +42,8 @@ void tsb_clear_range(struct hv_tsb_info *tsb, vm_offset_t sva, vm_offset_t eva); -void tsb_set_scratchpad(struct hv_tsb_info *tsb); +uint64_t tsb_set_scratchpad_kernel(struct hv_tsb_info *tsb); + +uint64_t tsb_set_scratchpad_user(struct hv_tsb_info *tsb); #endif /* !_MACHINE_TSB_H_ */ ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/include/tte_hash.h#4 (text+ko) ==== @@ -20,7 +20,9 @@ tte_t *tte_hash_lookup(tte_hash_t hash, vm_offset_t va); -void tte_hash_set_scratchpad(tte_hash_t th); +uint64_t tte_hash_set_scratchpad_kernel(tte_hash_t th); + +uint64_t tte_hash_set_scratchpad_user(tte_hash_t th); ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#21 (text+ko) ==== @@ -45,7 +45,9 @@ #include #include + #include "assym.s" +#include #define PMAP_DEBUG #if 1 @@ -220,7 +222,13 @@ .macro tl0_reserved count .rept \count +#if 0 tl0_gen T_RESERVED +#else + illtrap + .align 32 +#endif + .endr .endm @@ -255,6 +263,7 @@ .macro insn_excptn MAGIC_TRAP_ON + MAGIC_TRAP_ON illtrap .align 32 .endm @@ -888,7 +897,123 @@ add %g1, 1, %g1 ;\ wrpr %g1, %cwp - + + + +ENTRY(utl0) + SAVE_GLOBALS(%l7) + SAVE_OUTS(%l7) + wrpr %g0, PSTATE_KERNEL, %pstate ! enable ints + jmpl %l3, %o7 ! call trap handler + mov %l7, %o0 + +ENTRY(user_rtt) + ! pil handling needs to be re-visited + wrpr %g0, PIL_TICK, %pil + ldx [PCPU(CURTHREAD)], %l0 + lduw [%l0 + TD_FLAGS], %l1 + set TDF_ASTPENDING | TDF_NEEDRESCHED, %l2 + and %l1, %l2, %l1 + brz,a,pt %l1, 1f + nop + + ! handle AST and retry return + wrpr %g0, 0, %pil + call ast + mov %l7, %o0 + ba,a,pt %xcc, user_rtt + nop + +1: ldx [PCB_REG + PCB_NSAVED], %l1 + brz,a,pt %l1, 2f + nop + wrpr %g0, 0, %pil + mov T_SPILL, %o0 + stx %o0, [%l7 + SPOFF + CCFSZ + TF_TYPE] + call trap + mov %l7, %o0 + ba,a %xcc, user_rtt + nop +2: + ld [%sp + SPOFF + TF_WSTATE], %l3 + ! + ! restore user globals and outs + ! + rdpr %pstate, %l1 + wrpr %l1, PSTATE_IE, %pstate + RESTORE_GLOBALS(%l7) + wrpr %g0, 1, %gl + mov %sp, %g6 ! save tf pointer + RESTORE_OUTS(%l7) + + wrpr %g0, 0, %pil ! drop pil to 0 + + mov MMU_CID_S, %g1 + GET_MMU_CONTEXT(%g1, %g2) + mov MMU_CID_S, %g1 + sethi %hi(FLUSH_ADDR), %g3 + SET_MMU_CONTEXT(%g1, %g2) + flush %g3 ! flush required by immu + ! hangover from US I + ! + ! setup trap regs + ! + ldx [%l7 + TF_TPC], %g1 + ldx [%l7 + TF_TNPC], %g2 + ldx [%l7 + TF_TSTATE], %l0 + andn %l0, TSTATE_CWP_MASK, %g4 + wrpr %g1, %tpc + wrpr %g2, %tnpc + +#if 0 + ! + ! switch "other" windows back to "normal" windows and + ! restore to window we originally trapped in + ! + rdpr %otherwin, %g1 + wrpr %g0, 0, %otherwin + wrpr %g0, %g1, %canrestore +#endif + add %l3, WSTATE_CLEAN_OFFSET, %l3 ! convert to "clean" wstate + wrpr %g0, %l3, %wstate + + + rdpr %canrestore, %g1 + brnz %g1, 3f + nop ! no trap, use restore directly + rdpr %cwp, %g1 + wrpr %g1, %g7, %tstate ! needed by wbuf recovery code + ! hand craft the restore to avoid getting to TL > 2 + rdpr %wstate, %g1 + btst 1, %g1 + beq 4f + nop + .global rtt_fill_start +rtt_fill_start: +#if 0 + fill_32bit_rtt(ASI_AIUP) + ba,a 3f +#endif +4: + fill_64bit_rtt(ASI_AIUP) + .global rtt_fill_end +rtt_fill_end: +3: + restore ! should not trap +2: + + ! + ! set %cleanwin to %canrestore + ! set %tstate to the correct %cwp + ! retry resumes user execution + ! + rdpr %canrestore, %g1 + wrpr %g0, %g1, %cleanwin + rdpr %cwp, %g1 + wrpr %g1, %g7, %tstate + retry +END(user_rtt) +END(utl0) ENTRY(ktl0) SAVE_GLOBALS(%l7) @@ -1052,25 +1177,34 @@ MAGIC_TRAP_ON ldxa [%g1 + %g7]ASI_REAL, %g6 ! load in the context - srlx %g5, TTARGET_VA_SHIFT, %g1 - sllx %g6, TTARGET_CTX_SHIFT, %g6 - or %g6, %g1, %g6 ! %g6 == search tag + cmp %g6, %g0 ! kernel? + be,pn %xcc, 1f + srlx %g5, TTARGET_VA_SHIFT, %g1 + GET_HASH_SCRATCH_USER(%g2) + ba,pt %xcc, 2f + sllx %g6, TTARGET_CTX_SHIFT, %g6 +1: + GET_HASH_SCRATCH_KERNEL(%g2) + sllx %g6, TTARGET_CTX_SHIFT, %g6 +2: + or %g6, %g1, %g6 ! %g6 == search tag + + + ! %g4 == fault type %g5 == fault addr %g6 == tag ! XXX only handle normal miss for now (look at fault type in the future) #ifdef PMAP_DEBUG cmp %g6, %g0 ! NULL ptr deref in kernel - bne,pt %xcc, 4f + bne,pt %xcc, 3f nop tsb_miss_null_deref: illtrap ! give up -4: +3: #endif - mov 1, %g7 - sllx %g7, PAGE_SHIFT, %g7 + sethi %hi(PAGE_MASK), %g7 sub %g7, 1, %g7 ! %g7==PAGE_MASK - GET_HASH_SCRATCH(%g2) ! insn 2,3 and %g2, %g7, %g4 ! size stored in lower 13 bits andn %g2, %g7, %g2 ! actual VA of hash @@ -1093,57 +1227,58 @@ tsb_miss_lookup_0: ldda [%g2 + %g0]ASI_LDTD_N, %g4 cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 1f + be,pn %xcc, 4f nop cmp %g4, %g6 ! entry tag == VA tag? - be,pn %xcc, 2f + be,pn %xcc, 5f nop ! entry 1 tsb_miss_lookup_1: add %g2, 16, %g2 ! next THF ldda [%g2 + %g0]ASI_LDTD_N, %g4 cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 1f + be,pn %xcc, 4f nop cmp %g4, %g6 ! entry tag == search tag? - be,pn %xcc, 2f + be,pn %xcc, 5f nop ! entry 2 tsb_miss_lookup_2: add %g2, 16, %g2 ! next THF ldda [%g2 + %g0]ASI_LDTD_N, %g4 cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 1f + be,pn %xcc, 4f nop cmp %g4, %g6 ! entry tag == search tag? - be,pn %xcc, 2f + be,pn %xcc, 5f nop ! entry 3 tsb_miss_lookup_3: add %g2, 16, %g2 ! next THF ldda [%g2 + %g0]ASI_LDTD_N, %g4 cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 1f + be,pn %xcc, 4f nop cmp %g4, %g6 ! entry tag == search tag? - be,pn %xcc, 2f + be,pn %xcc, 5f nop tsb_miss_not_found: -1: ! not found +4: ! not found ! we need to jump to tl0_trap to drop us back down to tl0 ! and take us to trap(...) to service the fault ! skipping this step for the moment so we just do an illtrap - illtrap + ba %xcc, tl0_trap + mov T_DATA_MISS, %g1 tsb_miss_found: -2: !found +5: !found ! %g1 == abs index %g2 == THE pointer %g3 == flags ! %g4 == tag %g5 == data %g7 == PAGE_MASK ! %g3 <- TSB RA %g6 <- TSB size, TTE RA + srlx %g6, TTARGET_CTX_SHIFT, %g6 - ! XXX set referenced/modified bit unconditionally for now XXX andcc %g5, %g3, %g0 ! already set - bnz,pt %xcc, 5f + bnz,pt %xcc, 7f nop andcc %g3, VTD_REF, %g0 ! TSB miss bnz,pt %xcc, 6f @@ -1153,12 +1288,17 @@ or %g5, %g3, %g5 ! add ref/mod bit unconditionally 6: stx %g5, [%g2 + 8] ! set ref/mod bit -5: - GET_TSB_SCRATCH(%g3) ! %g3 == TSB (RA) - - +7: + cmp %g6, %g0 ! kernel context? + be,pn %xcc, 8f + nop + GET_TSB_SCRATCH_USER(%g3) ! %g3 == TSB (RA) + ba,a,pt %xcc, 9f + and %g3, %g7, %g6 ! size of TSB in pages +8: + GET_TSB_SCRATCH_KERNEL(%g3) ! %g3 == TSB (RA) and %g3, %g7, %g6 ! size of TSB in pages - +9: andn %g3, %g7, %g3 ! TSB real address sllx %g6, (PAGE_SHIFT - TTE_SHIFT), %g6 ! nttes subx %g6, 1, %g6 ! TSB_MASK @@ -1169,11 +1309,11 @@ #ifdef PMAP_DEBUG ldda [%g6]ASI_LDTD_REAL, %g2 cmp %g3, %g5 - bne,pt %xcc, 3f + bne,pt %xcc, 10f nop illtrap ! die if all we're doing ! is storing same data -3: +10: #endif stxa %g4, [%g6]ASI_REAL ! store tag stxa %g5, [%g6 + %g7]ASI_REAL ! store data @@ -1211,6 +1351,8 @@ ENTRY(prot_fault_trap) illtrap END(prot_fault_trap) + + /* * Freshly forked processes come here when switched to for the first time. * The arguments to fork_exit() have been setup in the locals, we must move @@ -1233,7 +1375,7 @@ mov %l1, %o1 call fork_exit mov %l2, %o2 - ba,a %xcc, tl0_ret + ba,a,pt %xcc, user_rtt nop END(fork_trampoline) ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/hcall.S#5 (text+ko) ==== @@ -1377,6 +1377,7 @@ bad_hcall_error: .asciz "hypervisor call 0x%x returned an unexpected error %d" + ENTRY(panic_bad_hcall) mov %o0, %o2 sethi %hi(bad_hcall_error), %o0 @@ -1398,6 +1399,21 @@ nop END(hv_magic_trap_off) + + ENTRY(hv_sim_read) + mov HVIO_SIM_READ, %o5 + ta FAST_TRAP + retl + nop + END(hv_read) + + ENTRY(hv_sim_write) + mov HVIO_SIM_WRITE, %o5 + ta FAST_TRAP + retl + nop + END(hv_write) + #endif /* lint || __lint */ ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/machdep.c#9 (text+ko) ==== ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/pmap.c#19 (text+ko) ==== @@ -354,7 +354,8 @@ oldpmap->pm_active &= ~1; pmap->pm_active |= 1; #endif - tte_hash_set_scratchpad(pmap->pm_hash); + pmap->pm_hashscratch = tte_hash_set_scratchpad_user(pmap->pm_hash); + pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb); PCPU_SET(curpmap, pmap); critical_exit(); } @@ -479,7 +480,7 @@ kernel_pmap->pm_tsb.hvtsb_rsvd = 0; kernel_pmap->pm_tsb.hvtsb_pa = pa; - tsb_set_scratchpad(&kernel_pmap->pm_tsb); + tsb_set_scratchpad_kernel(&kernel_pmap->pm_tsb); /* * Initialize kernel TSB for 4M pages @@ -620,7 +621,7 @@ * attempts to do updates until they're legal */ pm->pm_hash = tte_hash_kernel_create(kernel_hash, PAGE_SIZE_4M); - tte_hash_set_scratchpad(pm->pm_hash); + pm->pm_hashscratch = tte_hash_set_scratchpad_kernel(pm->pm_hash); for (i = 0; i < sz; i++) { if (translations[i].om_start < VM_MIN_PROM_ADDRESS || @@ -631,7 +632,7 @@ for (off = 0; off < translations[i].om_size; off += PAGE_SIZE) { va = translations[i].om_start + off; pa = TTE_GET_PA(translations[i].om_tte) + off; - tte_hash_insert(kernel_pmap->pm_hash, va, pa | TTE_KERNEL | VTD_8K); + tte_hash_insert(pm->pm_hash, va, pa | TTE_KERNEL | VTD_8K); } } } @@ -665,15 +666,12 @@ void pmap_clear_modify(vm_page_t m) { - /* XXX Need to also clear this in the TSB if possible :-( */ - UNIMPLEMENTED; tte_clear_phys_bit(m, VTD_W); } void pmap_clear_reference(vm_page_t m) { - UNIMPLEMENTED; tte_clear_phys_bit(m, VTD_REF); } @@ -1157,7 +1155,6 @@ pmap_pinit0(pmap_t pmap) { PMAP_LOCK_INIT(pmap); - pmap->pm_hash = NULL; pmap->pm_active = 0; pmap->pm_context = 0; PCPU_SET(curpmap, pmap); @@ -1180,6 +1177,7 @@ ("max context limit hit - need to implement context recycling")); pmap->pm_hash = tte_hash_create(pmap->pm_context); + pmap->pm_tsb_ra = tsb_init(&pmap->pm_tsb); pmap->pm_active = 0; TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1298,6 +1296,7 @@ pmap->pm_stats.resident_count)); pmap_lazyfix(pmap); + tsb_deinit(&pmap->pm_tsb); tte_hash_destroy(pmap->pm_hash); PMAP_LOCK_DESTROY(pmap); } ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/support.S#9 (text+ko) ==== @@ -353,7 +353,7 @@ * int copyin(const void *uaddr, void *kaddr, size_t len) */ ENTRY(copyin) - wr %g0, ASI_AIUP, %asi + wr %g0, ASI_AIUS, %asi _MEMCPY(%o1, %o0, %o2, E, E, a, %asi) retl clr %o0 @@ -363,7 +363,7 @@ * int copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done) */ ENTRY(copyinstr) - wr %g0, ASI_AIUP, %asi + wr %g0, ASI_AIUS, %asi _COPYSTR(%o0, %o1, %o2, %o3, a, %asi, E, E) retl mov %g1, %o0 @@ -373,7 +373,7 @@ * int copyout(const void *kaddr, void *uaddr, size_t len) */ ENTRY(copyout) - wr %g0, ASI_AIUP, %asi + wr %g0, ASI_AIUS, %asi _MEMCPY(%o1, %o0, %o2, a, %asi, E, E) retl clr %o0 @@ -412,7 +412,7 @@ * int32_t casuword32(volatile int32_t *p, int32_t e, int32_t s) */ ENTRY(casuword32) - casa [%o0] ASI_AIUP, %o1, %o2 + casa [%o0] ASI_AIUS, %o1, %o2 retl mov %o2, %o0 END(casuword32) @@ -421,7 +421,7 @@ * int64_t casuword64(volatile int64_t *p, int64_t e, int64_t s) */ ENTRY(casuword64) - casxa [%o0] ASI_AIUP, %o1, %o2 + casxa [%o0] ASI_AIUS, %o1, %o2 retl mov %o2, %o0 END(casuword64) @@ -431,7 +431,7 @@ */ ENTRY(fuword8) retl - lduba [%o0] ASI_AIUP, %o0 + lduba [%o0] ASI_AIUS, %o0 END(fuword8) /* @@ -439,7 +439,7 @@ */ ENTRY(fuword16) retl - lduha [%o0] ASI_AIUP, %o0 + lduha [%o0] ASI_AIUS, %o0 END(fuword16) /* @@ -447,7 +447,7 @@ */ ENTRY(fuword32) retl - lduwa [%o0] ASI_AIUP, %o0 + lduwa [%o0] ASI_AIUS, %o0 END(fuword32) /* @@ -455,14 +455,14 @@ */ ENTRY(fuword64) retl - ldxa [%o0] ASI_AIUP, %o0 + ldxa [%o0] ASI_AIUS, %o0 END(fuword64) /* * int suword8(const void *base, int word) */ ENTRY(suword8) - stba %o1, [%o0] ASI_AIUP + stba %o1, [%o0] ASI_AIUS retl clr %o0 END(suword8) @@ -471,7 +471,7 @@ * int suword16(const void *base, int word) */ ENTRY(suword16) - stha %o1, [%o0] ASI_AIUP + stha %o1, [%o0] ASI_AIUS retl clr %o0 END(suword16) @@ -480,7 +480,7 @@ * int suword32(const void *base, int32_t word) */ ENTRY(suword32) - stwa %o1, [%o0] ASI_AIUP + stwa %o1, [%o0] ASI_AIUS retl clr %o0 END(suword32) @@ -489,7 +489,7 @@ * int suword64(const void *base, int64_t word) */ ENTRY(suword64) - stxa %o1, [%o0] ASI_AIUP + stxa %o1, [%o0] ASI_AIUS retl clr %o0 END(suword64) @@ -503,14 +503,14 @@ */ ENTRY(fuswintr) retl - lduha [%o0] ASI_AIUP, %o0 + lduha [%o0] ASI_AIUS, %o0 END(fuswintr) /* * int suswintr(const void *base, int word) */ ENTRY(suswintr) - stha %o1, [%o0] ASI_AIUP + stha %o1, [%o0] ASI_AIUS retl clr %o0 END(suswintr) @@ -666,24 +666,37 @@ retl nop END(set_mmfsa_scratchpad) -ENTRY(set_hash_scratchpad) - mov SCRATCH_REG_HASH, %g2 +ENTRY(set_pcpu_scratchpad) + mov SCRATCH_REG_PCPU, %g2 + stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD + retl + nop +END(set_pcpu_scratchpad) +ENTRY(set_hash_kernel_scratchpad) + mov SCRATCH_REG_HASH_KERNEL, %g2 + stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD + retl + nop +END(set_hash_kernel_scratchpad) +ENTRY(set_tsb_kernel_scratchpad) + mov SCRATCH_REG_TSB_KERNEL, %g2 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD retl nop -END(set_hash_scratchpad) -ENTRY(set_pcpu_scratchpad) - mov SCRATCH_REG_PCPU, %g2 +END(set_tsb_kernel_scratchpad) + +ENTRY(set_hash_user_scratchpad) + mov SCRATCH_REG_HASH_USER, %g2 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD retl nop -END(set_pcpu_scratchpad) -ENTRY(set_tsb_scratchpad) - mov SCRATCH_REG_TSB, %g2 +END(set_hash_user_scratchpad) +ENTRY(set_tsb_user_scratchpad) + mov SCRATCH_REG_TSB_USER, %g2 stxa %o0, [%g0 + %g2]ASI_SCRATCHPAD retl nop -END(set_hash_scratchpad) +END(set_tsb_user_scratchpad) /* ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/swtch.S#8 (text+ko) ==== @@ -27,10 +27,11 @@ #include __FBSDID("$FreeBSD: src/sys/sparc64/sparc64/swtch.S,v 1.33 2004/05/26 12:06:52 tmm Exp $"); +#include #include -#include #include #include +#include #include "assym.s" @@ -122,21 +123,24 @@ stx %i1, [PCPU(CURPCB)] wrpr %g0, PSTATE_NORMAL, %pstate - mov %i1, PCB_REG + mov %i1, PCB_REG ! load in new PCB wrpr %g0, PSTATE_KERNEL, %pstate + /* + * End of saving old context + * Start of loading new context + */ ldx [PCB_REG + PCB_SP], %fp ldx [PCB_REG + PCB_PC], %i7 sub %fp, CCFSZ, %sp + /* * Point to the pmaps of the new process, and of the last non-kernel * process to run. */ ldx [%i0 + TD_PROC], %i2 -#if 0 ldx [PCPU(PMAP)], %l2 -#endif ldx [%i2 + P_VMSPACE], %i5 add %i5, VM_PMAP, %i2 @@ -147,108 +151,51 @@ stx %l2, [%g1 + KTR_PARM2] 9: #endif - /* * If they are the same we are done. */ - cmp %l2, %i2 + cmp %l2, %i2 ! current pmap == new pmap? be,a,pn %xcc, 5f - nop + nop - /* - * If the new process is a kernel thread we can just leave the old - * context active and avoid recycling its context number. - */ - SET(vmspace0, %i4, %i3) - cmp %i5, %i3 - be,a,pn %xcc, 5f - nop + ldx [%i2 + PM_CONTEXT], %l5 ! new context == kernel? + cmp %g0, %l5 + be,a,pn %xcc, 5f + nop - /* - * If there was no non-kernel pmap, don't try to deactivate it. - */ - brz,a,pn %l2, 3f - nop + MAGIC_TRAP_ON + mov SCRATCH_REG_HASH_USER, %l6 + mov SCRATCH_REG_TSB_USER, %l7 + ldx [%i2 + PM_HASHSCRATCH], %l3 + ldx [%i2 + PM_TSBSCRATCH], %l4 + stx %i2, [PCPU(PMAP)] - /* - * Mark the pmap of the last non-kernel vmspace to run as no longer - * active on this cpu. - */ - lduw [%l2 + PM_ACTIVE], %l3 - lduw [PCPU(CPUMASK)], %l4 - andn %l3, %l4, %l3 - stw %l3, [%l2 + PM_ACTIVE] + SET_SCRATCH(%l6,%l3) ! XXX we're assuming the + SET_SCRATCH(%l7,%l4) ! scratch values <= 32 bits - /* - * Take away its context number. - */ - lduw [PCPU(CPUID)], %l3 - sllx %l3, INT_SHIFT, %l3 - add %l2, PM_CONTEXT, %l4 - mov -1, %l5 - stw %l5, [%l3 + %l4] - /* - * Find a new tlb context. If we've run out we have to flush all user - * mappings from the tlb and reset the context numbers. - */ -3: lduw [PCPU(TLB_CTX)], %i3 - lduw [PCPU(TLB_CTX_MAX)], %i4 - cmp %i3, %i4 - bne,a,pt %xcc, 4f - nop - SET(invlctx, %i5, %i4) - ldx [%i4], %i5 - call %i5 - nop - lduw [PCPU(TLB_CTX_MIN)], %i3 + ldx [%i2 + PM_TSB_RA], %l3 - /* - * Advance next free context. - */ -4: add %i3, 1, %i4 - stw %i4, [PCPU(TLB_CTX)] + mov MMU_CID_S, %l6 - /* >>> TRUNCATED FOR MAIL (1000 lines) <<<