Date: Mon, 1 May 2006 01:31:30 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 96461 for review Message-ID: <200605010131.k411VUhE043958@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=96461 Change 96461 by kmacy@kmacy_storage:sun4v_rwbuf on 2006/05/01 01:31:10 add bucket locking in TSB miss handler simplify logic somewhat by using local registers Affected files ... .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#44 edit Differences ... ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#44 (text+ko) ==== @@ -260,7 +260,14 @@ .endm .macro insn_excptn - tl0_gen T_RESERVED + GET_MMFSA_SCRATCH(%g1) + mov MMFSA_D_ADDR, %g2 + ldxa [%g1 + %g2]ASI_REAL, %g3 + sub %g0, 1, %g4 + set trap, %g1 + ba %xcc, tl0_trap + mov T_INSTRUCTION_EXCEPTION, %g2 + .align 32 .endm @@ -310,12 +317,14 @@ .endm .macro tl0_align - GET_MMFSA_SCRATCH(%g1) + GET_MMFSA_SCRATCH(%g1) mov MMFSA_D_ADDR, %g3 - mov MMFSA_D_CTX, %g7 - ldxa [%g1 + %g3]ASI_REAL, %g5 +! mov MMFSA_D_CTX, %g7 + ldxa [%g1 + %g3]ASI_REAL, %g3 + ldxa [%g1 + %g7]ASI_REAL, %g4 + or %g4, %g3, %g3 ba,pt %xcc, tl0_trap - mov T_MEM_ADDRESS_NOT_ALIGNED, %g3 + mov T_MEM_ADDRESS_NOT_ALIGNED, %g2 .align 32 .endm @@ -330,6 +339,8 @@ .endm .macro resumable_error + MAGIC_TRAP_ON + MAGIC_EXIT clr %g3 sub %g0, 1, %g4 set trap, %g1 @@ -935,6 +946,7 @@ add %sp, REGOFF + SPOFF, %l7 ENTRY(user_rtt) GET_PCPU_SCRATCH + ! pil handling needs to be re-visited wrpr %g0, PIL_TICK, %pil ldx [PCPU(CURTHREAD)], %l4 @@ -964,6 +976,7 @@ ba,a %xcc, user_rtt nop 2: + ld [%l7 + TF_WSTATE], %l3 ! ! restore user globals and outs @@ -1321,9 +1334,33 @@ +#define HASH_LOOKUP(addr, tag, searchtag, endlabel, matchlabel) \ + ldda [addr]%asi, tag ; \ + cmp tag, searchtag ; \ + be,pn %xcc, matchlabel ; \ + nop ; \ + cmp tag, %g0 ; \ + be,pn %xcc, endlabel ; \ + nop +#define RESTORE_TRAPWIN(pcpu, cansave, label1, label2) \ + brz cansave, label1 ## f; \ + nop ; \ + mov -1, %l0 ; \ + mov -1, %l7 ; \ + restore ; \ + ba,a,pt %xcc, label2 ## f ; \ +label1: ; \ + rdpr %tl, cansave ; \ + dec cansave ; \ + sll cansave, RW_SHIFT, cansave ; \ + add cansave, PC_KWBUF, cansave ; \ + add pcpu, cansave, cansave ; \ + RESTORE_LOCALS_ASI(cansave) ; \ +label2: + ! %g3==trap type ! %g4==fault type (if data miss) ! %g5==fault addr @@ -1339,216 +1376,251 @@ ldxa [%g1 + %g7]ASI_REAL, %g6 ! load in the context rdpr %tl, %g7 ! need to use real addresses + +#ifdef notyet + rdpr %cansave, %g1 +#else + mov 0, %g1 +#endif + + brz,pn %g1, 0f + nop + save +0: + cmp %g7, 1 ! for tl > 1 bne,pn %xcc, 2f nop + mov ASI_LDTD_N, %g3 + wr %g0, ASI_N, %asi + GET_PCPU_SCRATCH + cmp %g6, %g0 ! kernel? be,pn %xcc, 1f nop GET_HASH_SCRATCH_USER(%g2) - wr %g0, ASI_LDTD_N, %asi - ba,pt %xcc, 4f + GET_TSB_SCRATCH_USER(%g4) + brz,pn %g1, 4f + nop + ba,pt %xcc, 5f nop 1: GET_HASH_SCRATCH_KERNEL(%g2) - wr %g0, ASI_LDTD_N, %asi - ba,pt %xcc, 4f + GET_TSB_SCRATCH_KERNEL(%g4) + brz,pn %g1, 4f + nop + ba,pt %xcc, 5f nop 2: + mov ASI_LDTD_REAL, %g3 + wr %g0, ASI_REAL, %asi + GET_PCPU_PHYS_SCRATCH(%g4) + cmp %g6, %g0 ! kernel? be,pn %xcc, 3f nop - GET_HASH_PHYS_SCRATCH_USER(%g7, %g2) - wr %g0, ASI_LDTD_REAL, %asi - ba,pt %xcc, 4f + GET_HASH_PHYS_SCRATCH_USER(%g4, %g2) + GET_TSB_SCRATCH_USER(%g4) + brz,pn %g1, 4f + nop + ba,pt %xcc, 5f + nop +3: + + GET_HASH_PHYS_SCRATCH_KERNEL(%g4, %g2) + GET_TSB_SCRATCH_KERNEL(%g4) + brnz,pt %g1, 5f nop -3: - GET_HASH_PHYS_SCRATCH_KERNEL(%g7, %g2) - wr %g0, ASI_LDTD_REAL, %asi -4: - - srlx %g5, TTARGET_VA_SHIFT, %g1 - sllx %g6, TTARGET_CTX_SHIFT, %g6 - or %g6, %g1, %g6 ! %g6 == search tag +4: + rdpr %tl, %g1 + dec %g1 + sll %g1, RW_SHIFT, %g1 + add %g1, PC_KWBUF, %g1 + add PCPU_REG, %g1, %g1 + SAVE_LOCALS_ASI(%g1) + mov 0, %g1 ! cansave is 0 +5: + + ! %g1 == %cansave + ! %g2 == hash scratch value + ! %g3 == TWDW ASI + ! %g4 == TSB RA + ! %g5 == fault addr + ! %g6 == context + srlx %g5, TTARGET_VA_SHIFT, %l0 + sllx %g6, TTARGET_CTX_SHIFT, %l1 + or %l0, %l1, %l2 ! %l2 == search tag - ! %g4 == fault type %g5 == fault addr %g6 == tag - ! XXX only handle normal miss for now (look at fault type in the future) - tsb_miss_compute_hash_addr: - sethi %hi(PAGE_SIZE), %g7 - sub %g7, 1, %g7 ! %g7==PAGE_MASK + sethi %hi(PAGE_SIZE), %l0 + sub %l0, 1, %l1 ! %l1==PAGE_MASK - and %g2, %g7, %g4 ! size stored in lower 13 bits - andn %g2, %g7, %g2 ! actual VA of hash + and %g2, %l1, %l3 ! size stored in lower 13 bits + andn %g2, %l1, %g2 ! actual VA/RA of hash ! XXX only handle 8k page miss ! calculate hash index - srlx %g5, PAGE_SHIFT, %g1 ! absolute hash index - sllx %g4, (PAGE_SHIFT - THE_SHIFT), %g4 ! size of hash in THEs - sub %g4, 1, %g4 ! THE_MASK - and %g1, %g4, %g4 ! masked hash index - sllx %g4, THE_SHIFT, %g4 ! masked hash offset + srlx %g5, PAGE_SHIFT, %l4 ! absolute hash index + sllx %l3, (PAGE_SHIFT - THE_SHIFT), %l0 ! size of hash in THEs + sub %l0, 1, %l5 ! THE_MASK + and %l4, %l5, %l5 ! masked hash index + sllx %l5, THE_SHIFT, %l5 ! masked hash offset ! fetch hash entries - exit when we find what were looking for ! %g2==entry base - add %g2, %g4, %g2 ! base + offset == entry base + add %g2, %l5, %g2 ! base + offset == entry base - mov %g5, %g7 ! save fault addr - ! entry 0 - ! %g1 == abs index %g2 == THE pointer %g3 == flags - ! %g4 <- tag %g5 <- data - ! %g6 == search tag %g7 == fault addr + mov 1, %l0 + add %g2, 8, %g2 ! point g2 at data + sllx %l0, 56, %l0 ! %l0 == VTD_LOCK +6: + ldxa [%g2]%asi, %l7 ! don't lock bus if bucket is locked + andcc %l7, %l0, %g0 + bnz,pn %xcc, 6b + or %l7, %l0, %l6 + casxa [%g2]%asi, %l7, %l6 + cmp %l6, %l7 + bne,pn %icc, 6b + nop + + sub %g2, 8, %g2 ! point g2 back at tag + + ! %g1 == cansave + ! %g2 == THE + ! %g3 == TWDW ASI + ! %g4 == TSB RA + ! %g5 == fault addr + ! %g6 == context + ! %g7 == PCPU_REG + ! %l0 == VTD_LOCK + ! %l1 == PAGE_MASK + ! %l2 == search tag + ! %l4 == absolute index + + ! %l3 == ASI + ! %l5 == saved head of bucket + ! %l6 == tag + ! %l7 == data + + rd %asi, %l3 + wr %g0, %g3, %asi + mov %g2, %l5 ! save head of bucket + rdpr %tt, %g3 ! reload trap type -tsb_miss_lookup_0: - ldda [%g2]%asi, %g4 - cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 5f - nop - cmp %g4, %g6 ! entry tag == VA tag? - be,pn %xcc, 6f - nop - ! entry 1 +tsb_miss_lookup_0: + HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f) tsb_miss_lookup_1: add %g2, 16, %g2 - ldda [%g2]%asi, %g4 - cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 5f - nop - cmp %g4, %g6 ! entry tag == search tag? - be,pn %xcc, 6f - nop - ! entry 2 + HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f) tsb_miss_lookup_2: add %g2, 16, %g2 - ldda [%g2]%asi, %g4 - cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 5f - nop - cmp %g4, %g6 ! entry tag == search tag? - be,pn %xcc, 6f - nop - ! entry 3 + HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f) tsb_miss_lookup_3: add %g2, 16, %g2 - ldda [%g2]%asi, %g4 - cmp %g4, %g0 ! entry tag == 0 ? - be,pn %xcc, 5f - nop - cmp %g4, %g6 ! entry tag == search tag? - be,pn %xcc, 6f - nop + HASH_LOOKUP(%g2, %l6, %l2, 7f, 8f) tsb_miss_not_found: -5: ! not found +7: ! not found ! we need to jump to tl0_trap to drop us back down to tl0 ! and take us to trap(...) to service the fault + wr %g0, %l3, %asi + + ldxa [%l5 + 8]%asi, %g2 + andn %g2, %l0, %g2 + stxa %g2, [%l5 + 8]%asi ! unlock head of bucket - srlx %g7, 13, %g7 ! slow painful way of masking off - sllx %g7, 13, %g7 ! bottom bits without using a reg - ! XXX FIXME - - mov %g3, %g2 ! save fault type - srlx %g6, TTARGET_CTX_SHIFT, %g4 ! recover context + andn %g5, %l1, %g5 ! fault page PA + + RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15) + + mov %g3, %g2 ! trap type sethi %hi(trap), %g1 - or %g4, %g7, %g3 - mov -1, %g4 + or %g6, %g5, %g3 ! trap data + sub %g0, 1, %g4 ! pil info ba %xcc, tl0_trap or %g1, %lo(trap), %g1 - -tsb_miss_found: -6: !found - ! %g1 == abs index %g2 == THE pointer %g3 == flags - ! %g4 == tag %g5 == data %g7 == PAGE_MASK - ! %g3 <- TSB RA %g6 <- TSB size, TTE RA %g7 == PAGE_MASK - srlx %g6, TTARGET_CTX_SHIFT, %g6 - or %g7, %g6, %g6 ! save context and fault addr - ! will break when ctxbits > 13 - sethi %hi(PAGE_SIZE), %g7 - sub %g7, 1, %g7 ! %g7==PAGE_MASK - +tsb_miss_found: +8: + wr %g0, %l3, %asi cmp %g3, T_DATA_MISS ! TSB data miss - be,pt %xcc, 7f - or %g5, VTD_REF, %g5 ! set referenced unconditionally + be,pt %xcc, 9f + or %l7, VTD_REF, %l7 ! set referenced unconditionally cmp %g3, T_INSTRUCTION_MISS ! TSB instruction miss - be,pt %xcc, 7f + be,pt %xcc, 9f nop cmp %g3, T_DATA_PROTECTION ! protection fault bne,pn %xcc, unsupported_fault_trap ! we don't handle any other fault types currently nop - andcc %g5, VTD_SW_W, %g0 ! write enabled? - bz,pn %xcc, prot_fault_trap ! write to read only page + andcc %l7, VTD_SW_W, %g0 ! write enabled? + bz,a,pn %xcc, prot_fault_trap ! write to read only page nop - or %g5, VTD_W, %g5 ! add modifed bit -7: - rdpr %tl, %g3 - dec %g3 - movrnz %g3, ASI_REAL, %g3 - movrz %g3, ASI_N, %g3 - wr %g0, %g3, %asi + or %l7, VTD_W, %l7 ! add modifed bit +9: + + + andn %l7, %l0, %l7 ! remove lock bit + + and %g4, %l1, %g3 ! size of TSB in pages + andn %g4, %l1, %l3 ! TSB real address + + sllx %g3, (PAGE_SHIFT - TTE_SHIFT), %g3 ! nttes + subx %g3, 1, %g3 ! TSB_MASK + and %g3, %l4, %g3 ! masked index + sllx %g3, TTE_SHIFT, %g3 ! masked byte offset + add %g3, %l3, %g3 ! TTE RA + + mov 8, %l4 - stxa %g5, [%g2 + 8]%asi ! update TTE - andcc %g6, %g7, %g0 ! kernel context? - bz,pn %xcc, 8f - nop - GET_TSB_SCRATCH_USER(%g3) ! %g3 == TSB (RA) - ba,pt %xcc, 9f - and %g3, %g7, %g2 ! size of TSB in pages -8: - GET_TSB_SCRATCH_KERNEL(%g3) ! %g3 == TSB (RA) - and %g3, %g7, %g2 ! size of TSB in pages -9: - andn %g3, %g7, %g3 ! TSB real address - sllx %g2, (PAGE_SHIFT - TTE_SHIFT), %g2 ! nttes - subx %g2, 1, %g2 ! TSB_MASK - and %g2, %g1, %g2 ! masked index - sllx %g2, TTE_SHIFT, %g2 ! masked byte offset - add %g2, %g3, %g2 ! TTE RA - mov 8, %g7 #ifdef PMAP_DEBUG - mov %g2, %g1 - ldda [%g2]ASI_LDTD_REAL, %g2 - cmp %g3, %g5 - bne,pt %xcc, 10f - cmp %g2, %g4 - bne,pt %xcc, 10f + ldda [%g3]ASI_LDTD_REAL, %l2 + cmp %l3, %l7 + bne,pt %xcc, 12f + cmp %l2, %l6 + bne,pt %xcc, 12f nop +#ifndef SIMULATOR PUTCHAR(0x5a) PUTCHAR(0x5a) PUTCHAR(0x5a) +#endif MAGIC_TRAP_ON;MAGIC_TRAP_ON;MAGIC_EXIT ! die if all we're doing ! is storing same data -10: - mov %g1, %g2 +12: #endif - stxa %g0, [%g2 + %g7]ASI_REAL ! invalidate data - stxa %g4, [%g2]ASI_REAL ! store tag - stxa %g5, [%g2 + %g7]ASI_REAL ! store data + stxa %g0, [%g3 + %l4]ASI_REAL ! invalidate data + stxa %l6, [%g3]ASI_REAL ! store tag + stxa %l7, [%g3 + %l4]ASI_REAL ! store data + stxa %l7, [%g2 + 8]%asi ! update TTE -upgrade_demap: - MAGIC_TRAP_ON - rdpr %tt, %g5 - cmp %g5, T_DATA_PROTECTION + cmp %g2, %l5 + beq,a,pt %xcc, 17f + nop + ldxa [%l5 + 8]%asi, %l7 + andn %l7, %l0, %l7 ! remove lock bit + stxa %l7, [%l5 + 8]%asi ! unlock bucket +17: + membar #Sync + RESTORE_TRAPWIN(PCPU_REG, %g1, 13, 16) +upgrade_demap: + rdpr %tt, %g3 + cmp %g3, T_DATA_PROTECTION beq,pn %xcc, demap_begin nop - MAGIC_TRAP_OFF retry demap_begin: sethi %hi(PAGE_SIZE), %g1 sub %g1, 1, %g1 - and %g6, %g1, %g2 - andn %g6, %g1, %g3 - mov %o0, %g5 - mov %o1, %g6 - mov %o2, %g7 + mov %o0, %g1 + mov %o1, %g2 + mov %o2, %g3 mov MAP_DTLB, %o2 - mov %g3, %o0 - mov %g2, %o1 + mov %g5, %o0 + mov %g6, %o1 ta MMU_UNMAP_ADDR - mov %g5, %o0 - mov %g6, %o1 - mov %g7, %o2 - MAGIC_TRAP_OFF + mov %g1, %o0 + mov %g2, %o1 + mov %g3, %o2 retry END(tsb_miss_handler) @@ -1556,13 +1628,21 @@ /* * Write to read-only page */ -! %g4==tag -! %g5==TTE -! %g6==fault addr | context +! %g1 == cansave +! %g4 == tag +! %g5 == fault addr +! %g6 == context +! %l0 == VTD_LOCK +! %l5 == head of bucket + ENTRY(prot_fault_trap) + ldxa [%l5 + 8]%asi, %g2 + andn %g2, %l0, %g2 ! remove lock bit + stxa %g2, [%l5 + 8]%asi ! unlock head of bucket + RESTORE_TRAPWIN(PCPU_REG, %g1, 14, 15) sethi %hi(trap), %g1 mov T_DATA_PROTECTION, %g2 - mov %g6, %g3 + or %g5, %g6, %g3 sub %g0, 1, %g4 ba %xcc, tl0_trap or %g1, %lo(trap), %g1
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200605010131.k411VUhE043958>