Date: Sat, 18 Feb 2006 02:57:37 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 91975 for review Message-ID: <200602180257.k1I2vb8m083029@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=91975 Change 91975 by kmacy@kmacy_storage:sun4v_work on 2006/02/18 02:57:18 flesh out initial register spill-fill handling code remove rwindow.c as its rwindow_load / rwindow_store as it is all code that should be handled in assembler Affected files ... .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#8 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/locore.S#4 edit .. //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/rwindow.c#2 delete Differences ... ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/exception.S#8 (text+ko) ==== @@ -101,47 +101,79 @@ /* * Macros for spilling and filling live windows. + * Here we use the more complicated [regaddr] format which requires + * us to interleave setting the globals in order to be able to use + * imm_asi - we don't ever implicitly assume kernel context as in + * Solaris' spill/fill handlers so that we have the option of using + * block initializing stores - twin doubleword loads could also be + * advantageous but will require an additional macro * - * NOTE: These macros use exactly 16 instructions, and it is assumed that the - * handler will not use more than 24 instructions total, to leave room for - * resume vectors which occupy the last 8 instructions. */ -#define SPILL(storer, base, size, asi) \ - storer %l0, [base + (0 * size)] asi ; \ - storer %l1, [base + (1 * size)] asi ; \ - storer %l2, [base + (2 * size)] asi ; \ - storer %l3, [base + (3 * size)] asi ; \ - storer %l4, [base + (4 * size)] asi ; \ - storer %l5, [base + (5 * size)] asi ; \ - storer %l6, [base + (6 * size)] asi ; \ - storer %l7, [base + (7 * size)] asi ; \ - storer %i0, [base + (8 * size)] asi ; \ - storer %i1, [base + (9 * size)] asi ; \ - storer %i2, [base + (10 * size)] asi ; \ - storer %i3, [base + (11 * size)] asi ; \ - storer %i4, [base + (12 * size)] asi ; \ - storer %i5, [base + (13 * size)] asi ; \ - storer %i6, [base + (14 * size)] asi ; \ - storer %i7, [base + (15 * size)] asi +#define SPILL(storer, bias, size, asi) \ + mov 0 + bias, %g1 ;\ + storer %l0, [%sp + %g1]asi ;\ + mov size + bias, %g2 ;\ + storer %l1, [%sp + %g2]asi ;\ + mov (2 * size) + bias, %g3 ;\ + storer %l2, [%sp + %g3]asi ;\ + mov (3 * size) + bias, %g4 ;\ + storer %l3, [%sp + %g4]asi ;\ + add %sp, (4 * size), %g5 ;\ + storer %l4, [%g5 + %g1]asi ;\ + storer %l5, [%g5 + %g2]asi ;\ + storer %l6, [%g5 + %g3]asi ;\ + storer %l7, [%g5 + %g4]asi ;\ + add %sp, (4 * size), %g5 ;\ + storer %i0, [%g5 + %g1]asi ;\ + storer %i1, [%g5 + %g2]asi ;\ + storer %i2, [%g5 + %g3]asi ;\ + storer %i3, [%g5 + %g4]asi ;\ + add %sp, (4 * size), %g5 ;\ + storer %i4, [%g5 + %g1]asi ;\ + storer %i5, [%g5 + %g2]asi ;\ + storer %i6, [%g5 + %g3]asi ;\ + storer %i7, [%g5 + %g4]asi + +#define FILL(loader, bias, size, asi) \ + mov 0 + bias, %g1 ;\ + loader [%sp + %g1]asi, %l0 ;\ + mov size + bias, %g2 ;\ + loader [%sp + %g2]asi, %l1 ;\ + mov (2 * size) + bias, %g3 ;\ + loader [%sp + %g3]asi, %l2 ;\ + mov (3 * size) + bias, %g4 ;\ + loader [%sp + %g4]asi, %l3 ;\ + add %sp, (4 * size), %g5 ;\ + loader [%g5 + %g1]asi, %l4 ;\ + loader [%g5 + %g2]asi, %l5 ;\ + loader [%g5 + %g3]asi, %l6 ;\ + loader [%g5 + %g4]asi, %l7 ;\ + add %sp, (4 * size), %g5 ;\ + loader [%g5 + %g1]asi, %i0 ;\ + loader [%g5 + %g2]asi, %i1 ;\ + loader [%g5 + %g3]asi, %i2 ;\ + loader [%g5 + %g4]asi, %i3 ;\ + add %sp, (4 * size), %g5 ;\ + loader [%g5 + %g1]asi, %i4 ;\ + loader [%g5 + %g2]asi, %i5 ;\ + loader [%g5 + %g3]asi, %i6 ;\ + loader [%g5 + %g4]asi, %i7 -#define FILL(loader, base, size, asi) \ - loader [base + (0 * size)] asi, %l0 ; \ - loader [base + (1 * size)] asi, %l1 ; \ - loader [base + (2 * size)] asi, %l2 ; \ - loader [base + (3 * size)] asi, %l3 ; \ - loader [base + (4 * size)] asi, %l4 ; \ - loader [base + (5 * size)] asi, %l5 ; \ - loader [base + (6 * size)] asi, %l6 ; \ - loader [base + (7 * size)] asi, %l7 ; \ - loader [base + (8 * size)] asi, %i0 ; \ - loader [base + (9 * size)] asi, %i1 ; \ - loader [base + (10 * size)] asi, %i2 ; \ - loader [base + (11 * size)] asi, %i3 ; \ - loader [base + (12 * size)] asi, %i4 ; \ - loader [base + (13 * size)] asi, %i5 ; \ - loader [base + (14 * size)] asi, %i6 ; \ - loader [base + (15 * size)] asi, %i7 +#define FILL_DW(asi) \ + mov 0 + SPOFF, %g1 ;\ + ldda [%sp + %g1]asi, %l0 ;\ + mov 16 + SPOFF, %g2 ;\ + ldda [%sp + %g2]asi, %l2 ;\ + add %sp, 32, %g3 ;\ + ldda [%g3 + %g1]asi, %l4 ;\ + ldda [%g3 + %g2]asi, %l6 ;\ + add %sp, 32, %g3 ;\ + ldda [%g3 + %g1]asi, %i0 ;\ + ldda [%g3 + %g2]asi, %i2 ;\ + add %sp, 32, %g3 ;\ + ldda [%g3 + %g1]asi, %i4 ;\ + ldda [%g3 + %g2]asi, %i6 #define ERRATUM50(reg) mov reg, reg @@ -314,41 +346,7 @@ retry ; \ .align 16 -/* - * Generate a T_SPILL or T_FILL trap if the window operation fails. - */ -#define RSF_TRAP(type) \ - ba %xcc, tl0_sftrap ; \ - mov type, %g2 ; \ - .align 16 -/* - * Game over if the window operation fails. - */ -#define RSF_FATAL(type) \ - ba %xcc, rsf_fatal ; \ - mov type, %g2 ; \ - .align 16 - -/* - * Magic to resume from a failed fill a few instructions after the corrsponding - * restore. This is used on return from the kernel to usermode. - */ -#define RSF_FILL_MAGIC \ - rdpr %tnpc, %g1 ; \ - add %g1, RSF_FILL_INC, %g1 ; \ - wrpr %g1, 0, %tnpc ; \ - done ; \ - .align 16 - -/* - * Spill to the pcb if a spill to the user stack in kernel mode fails. - */ -#define RSF_SPILL_TOPCB \ - ba,a %xcc, tl1_spill_topcb ; \ - nop ; \ - .align 16 - ENTRY(rsf_fatal) #if KTR_COMPILE & KTR_TRAP CATR(KTR_TRAP, "rsf_fatal: bad window trap tt=%#lx type=%#lx" @@ -523,65 +521,148 @@ .align 32 .endm - .macro spill_32bit_asi asi target + + +#define spill_32bit_asi(asi, target) \ + srl %sp, 0, %sp ; \ + SPILL(sta, 0, 4, asi) ; \ + saved ; \ + retry ; \ + .skip (31-26)*4 ; \ + ba,a,pt %xcc, fault_32bit_##target ; \ + .align 128 + +#define spill_64bit_asi(asi, target) \ + SPILL(stxa, SPOFF, 8, asi) ; \ + saved ; \ + retry ; \ + .skip (31-25)*4 ; \ + ba,a,pt %xcc, fault_64bit_##target ; \ + .align 128 + +#define spill_32clean(asi, target) \ + srl %sp, 0, %sp ; \ + SPILL(sta, 0, 4, asi) ; \ + b spill_clean ; \ + mov WSTATE_USER32, %g7 ; \ + .skip (31-26)*4 ; \ + ba,a,pt %xcc, fault_32bit_##target ; \ .align 128 + +#define spill_64clean(asi, target) \ + SPILL(stxa, SPOFF, 8, asi) ; \ + b spill_clean ; \ + mov WSTATE_USER64, %g7 ; \ + .skip (31-25)*4 ; \ + ba,a,pt %xcc, fault_64bit_##target ; \ + .align 128 + +#define fill_32bit_asi(asi, target) \ + srl %sp, 0, %sp ; \ + FILL(lda, 0, 4, asi) ; \ + saved ; \ + retry ; \ + .skip (31-26)*4 ; \ + ba,a,pt %xcc, fault_32bit_##target ; \ + .align 128 + +#define fill_64bit_asi(asi, target) \ + FILL(ldxa, SPOFF, 8, asi) ; \ + saved ; \ + retry ; \ + .skip (31-25)*4 ; \ + ba,a,pt %xcc, fault_64bit_##target ; \ + .align 128 + + .macro spill_32bit_primary_sn0 + spill_32bit_asi(ASI_AIUP, sn0) + .endm + + .macro spill_64bit_primary_sn0 + spill_64bit_asi(ASI_AIUP, sn0) .endm - .macro spill_64bit_asi asi target - .align 128 + .macro spill_32clean_primary_sn0 + spill_32clean(ASI_AIUP, sn0) + .endm + + .macro spill_64clean_primary_sn0 + spill_64clean(ASI_AIUP, sn0) .endm - .macro spill_32bit target - .align 128 + .macro spill_32bit_nucleus_not + spill_32bit_asi(ASI_N, not) .endm - .macro spill_64bit target - .align 128 + .macro spill_64bit_nucleus_not + spill_64bit_asi(ASI_N,not) .endm - .macro spill_64bit_ktt1 target - .align 128 + .macro spill_32bit_secondary_so0 + spill_32bit_asi(ASI_AIUS, so0) .endm - .macro spill_mixed_ktt1 target - .align 128 + .macro spill_64bit_secondary_so0 + spill_64bit_asi(ASI_AIUS, so0) + .endm + + .macro fill_32bit_primary_fn0 + fill_32bit_asi(ASI_AIUP, fn0) .endm - .macro spill_32bit_tt1 asi target - .align 128 + .macro fill_64bit_primary_fn0 + fill_64bit_asi(ASI_AIUP, fn0) .endm - .macro spill_64bit_tt1 asi target - .align 128 + .macro fill_32bit_nucleus_not + fill_32bit_asi(ASI_N, not) .endm - .macro spill_32clean asi target - .align 128 + .macro fill_64bit_nucleus_not + fill_64bit_asi(ASI_N, not) .endm - .macro spill_64clean asi target - .align 128 + .macro spill_32bit_tt1_primary_sn1 + ba,a,pt %xcc, fault_32bit_sn1 + nop + .align 128 .endm - .macro spill_mixed - .align 128 + .macro spill_64bit_tt1_primary_sn1 + ba,a,pt %xcc, fault_64bit_sn1 + nop + .align 128 .endm + .macro spill_64bit_ktt1_sk + ba,a,pt %xcc, fault_64bit_sk + nop + .align 128 + .endm - - .macro fill_32bit_asi asi target - .align 128 + .macro spill_mixed_ktt1_sk + btst 1, %sp + bz,a,pt %xcc, fault_32bit_sk + srl %sp, 0, %sp + ba,a,pt %xcc, fault_64bit_sk + nop + .align 128 .endm - .macro fill_64bit_asi asi target - .align 128 + .macro spill_32bit_tt1_secondary_so1 + ba,a,pt %xcc, fault_32bit_so1 + nop + .align 128 .endm - .macro fill_32bit target - .align 128 + .macro spill_64bit_tt1_secondary_so1 + ba,a,pt %xcc, fault_64bit_so1 + nop + .align 128 + .endm - .macro fill_64bit target + .macro spill_mixed .align 128 .endm @@ -1064,45 +1145,6 @@ mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl0_dmmu_prot_trap) - .macro tl0_spill_0_n - wr %g0, ASI_AIUP, %asi - SPILL(stxa, %sp + SPOFF, 8, %asi) - saved - retry - .align 32 - RSF_TRAP(T_SPILL) - RSF_TRAP(T_SPILL) - .endm - - .macro tl0_spill_1_n - wr %g0, ASI_AIUP, %asi - SPILL(stwa, %sp, 4, %asi) - saved - retry - .align 32 - RSF_TRAP(T_SPILL) - RSF_TRAP(T_SPILL) - .endm - - .macro tl0_fill_0_n - wr %g0, ASI_AIUP, %asi - FILL(ldxa, %sp + SPOFF, 8, %asi) - restored - retry - .align 32 - RSF_TRAP(T_FILL) - RSF_TRAP(T_FILL) - .endm - - .macro tl0_fill_1_n - wr %g0, ASI_AIUP, %asi - FILL(lduwa, %sp, 4, %asi) - restored - retry - .align 32 - RSF_TRAP(T_FILL) - RSF_TRAP(T_FILL) - .endm #endif /* notyet */ ENTRY(tl0_sftrap) rdpr %tstate, %g1 @@ -1602,142 +1644,8 @@ mov T_DATA_PROTECTION | T_KERNEL, %o0 END(tl1_dmmu_prot_trap) #endif /* notyet */ - .macro tl1_spill_0_n - SPILL(stx, %sp + SPOFF, 8, EMPTY) - saved - retry - .align 32 - RSF_FATAL(T_SPILL) - RSF_FATAL(T_SPILL) - .endm - - .macro tl1_spill_2_n - wr %g0, ASI_AIUP, %asi - SPILL(stxa, %sp + SPOFF, 8, %asi) - saved - retry - .align 32 - RSF_SPILL_TOPCB - RSF_SPILL_TOPCB - .endm - - .macro tl1_spill_3_n - wr %g0, ASI_AIUP, %asi - SPILL(stwa, %sp, 4, %asi) - saved - retry - .align 32 - RSF_SPILL_TOPCB - RSF_SPILL_TOPCB - .endm - - .macro tl1_spill_0_o - wr %g0, ASI_AIUP, %asi - SPILL(stxa, %sp + SPOFF, 8, %asi) - saved - retry - .align 32 - RSF_SPILL_TOPCB - RSF_SPILL_TOPCB - .endm - - .macro tl1_spill_1_o - wr %g0, ASI_AIUP, %asi - SPILL(stwa, %sp, 4, %asi) - saved - retry - .align 32 - RSF_SPILL_TOPCB - RSF_SPILL_TOPCB - .endm - - .macro tl1_spill_2_o - RSF_SPILL_TOPCB - .align 128 - .endm - - .macro tl1_fill_0_n - FILL(ldx, %sp + SPOFF, 8, EMPTY) - restored - retry - .align 32 - RSF_FATAL(T_FILL) - RSF_FATAL(T_FILL) - .endm - - .macro tl1_fill_2_n - wr %g0, ASI_AIUP, %asi - FILL(ldxa, %sp + SPOFF, 8, %asi) - restored - retry - .align 32 - RSF_FILL_MAGIC - RSF_FILL_MAGIC - .endm - - .macro tl1_fill_3_n - wr %g0, ASI_AIUP, %asi - FILL(lduwa, %sp, 4, %asi) - restored - retry - .align 32 - RSF_FILL_MAGIC - RSF_FILL_MAGIC - .endm - -/* - * This is used to spill windows that are still occupied with user - * data on kernel entry to the pcb. - */ -ENTRY(tl1_spill_topcb) -#ifdef notyet - /* XXX SUN4V_FIXME - I think this broken from a performance - * standpoint - */ - LOAD_ALT - - /* Free some globals for our use. */ - dec 24, ASP_REG - stx %g1, [ASP_REG + 0] - stx %g2, [ASP_REG + 8] - stx %g3, [ASP_REG + 16] - - ldx [PCB_REG + PCB_NSAVED], %g1 - - sllx %g1, PTR_SHIFT, %g2 - add %g2, PCB_REG, %g2 - stx %sp, [%g2 + PCB_RWSP] - - sllx %g1, RW_SHIFT, %g2 - add %g2, PCB_REG, %g2 - SPILL(stx, %g2 + PCB_RW, 8, EMPTY) - - inc %g1 - stx %g1, [PCB_REG + PCB_NSAVED] -#if KTR_COMPILE & KTR_TRAP - CATR(KTR_TRAP, "tl1_spill_topcb: pc=%#lx npc=%#lx sp=%#lx nsaved=%d" - , %g1, %g2, %g3, 7, 8, 9) - rdpr %tpc, %g2 - stx %g2, [%g1 + KTR_PARM1] - rdpr %tnpc, %g2 - stx %g2, [%g1 + KTR_PARM2] - stx %sp, [%g1 + KTR_PARM3] - ldx [PCB_REG + PCB_NSAVED], %g2 - stx %g2, [%g1 + KTR_PARM4] -9: -#endif - - saved - ldx [ASP_REG + 16], %g3 - ldx [ASP_REG + 8], %g2 - ldx [ASP_REG + 0], %g1 - inc 24, ASP_REG - retry -END(tl1_spill_topcb) -#endif - .macro tl1_soft count .rept \count tl1_gen T_SOFT | T_KERNEL @@ -1808,28 +1716,28 @@ nonresumable_error ! 0x7f tl0_spill_n_normal: tl0_reserved 4 ! 0x80 - spill_32bit_asi ASI_AIUP sn0 ! 0x84 - spill_64bit_asi ASI_AIUP sn0 ! 0x88 - ASI_BLK_AIUP - spill_32clean ASI_AIUP sn0 ! 0x8c - spill_64clean ASI_AIUP sn0 ! 0x90 - ASI_BLK_AIUP - spill_32bit not ! 0x94 - spill_64bit not ! 0x98 + spill_32bit_primary_sn0 ! 0x84 + spill_64bit_primary_sn0 ! 0x88 + spill_32clean_primary_sn0 ! 0x8c + spill_64clean_primary_sn0 ! 0x90 + spill_32bit_nucleus_not ! 0x94 + spill_64bit_nucleus_not ! 0x98 spill_mixed ! 0x9c tl0_spill_n_other: tl0_reserved 4 ! 0xa0 - spill_32bit_asi ASI_AIUS so0 ! 0xa4 - spill_64bit_asi ASI_AIUS so0 ! 0xa8 - ASI_BLK_AIUS - spill_32bit_asi ASI_AIUS so0 ! 0xac - spill_64bit_asi ASI_AIUS so0 ! 0xb0 - ASI_BLK_AIUS + spill_32bit_secondary_so0 ! 0xa4 + spill_64bit_secondary_so0 ! 0xa8 + spill_32bit_secondary_so0 ! 0xac + spill_64bit_secondary_so0 ! 0xb0 tl0_reserved 12 ! 0xb4-0xbf tl0_fill_n_normal: tl0_reserved 4 ! 0xa0 - fill_32bit_asi ASI_AIUP fn0 ! 0xc4 - fill_64bit_asi ASI_AIUP fn0 ! 0xc8 - ASI_LDTD_AIUP - (requires rewrite) - fill_32bit_asi ASI_AIUP fn0 ! 0xcc - fill_64bit_asi ASI_AIUP fn0 ! 0xd0 - ASI_LDTD_AIUP - (requires rewrite) - fill_32bit not ! 0xd4 - fill_64bit not ! 0xd8 + fill_32bit_primary_fn0 ! 0xc4 + fill_64bit_primary_fn0 ! 0xc8 + fill_32bit_primary_fn0 ! 0xcc + fill_64bit_primary_fn0 ! 0xd0 + fill_32bit_nucleus_not ! 0xd4 + fill_64bit_nucleus_not ! 0xd8 fill_mixed ! 0xdc tl0_fill_n_other: tl0_reserved 32 ! 0xe0-0xff @@ -1891,26 +1799,49 @@ nonresumable_error ! 0x27f tl1_spill_n_normal: tl1_reserved 4 ! 0x280 - spill_32bit_tt1 ASI_AIUP sn1 ! 0x284 - spill_64bit_tt1 ASI_AIUP sn1 ! 0x288 - ASI_BLK_AIUP - spill_32bit_tt1 ASI_AIUP sn1 ! 0x28c - spill_64bit_tt1 ASI_AIUP sn1 ! 0x290 - ASI_BLK_AIUP + spill_32bit_tt1_primary_sn1 ! 0x284 + spill_64bit_tt1_primary_sn1 ! 0x288 + spill_32bit_tt1_primary_sn1 ! 0x28c + spill_64bit_tt1_primary_sn1 ! 0x290 tl1_reserved ! 0x294 - spill_64bit_ktt1 sk ! 0x298 - spill_mixed_ktt1 sk ! 0x29c + spill_64bit_ktt1_sk ! 0x298 + spill_mixed_ktt1_sk ! 0x29c tl1_spill_n_other: tl1_reserved 4 ! 0x2a0 - spill_32bit_tt1 ASI_AIUS so1 ! 0x2a4 - spill_64bit_tt1 ASI_AIUS so1 ! 0x2a8 - ASI_BLK_AIUS - spill_32bit_tt1 ASI_AIUS so1 ! 0x2ac - spill_64bit_tt1 ASI_AIUS so1 ! 0x2b0 - ASI_BLK_AIUS + spill_32bit_tt1_secondary_so1 ! 0x2a4 + spill_64bit_tt1_secondary_so1 ! 0x2a8 + spill_32bit_tt1_secondary_so1 ! 0x2ac + spill_64bit_tt1_secondary_so1 ! 0x2b0 tl1_reserved 12 ! 0x2b4-0x2bf tl1_fill_n_normal: tl1_reserved 32 ! 0x2c0-0x2df tl1_fill_n_other: tl1_reserved 32 ! 0x2e0-0x2ff +/* + * no discretionary traps at TL 1 - leaving us with a 24k trap table + */ +spill_clean: + sethi %hi(nwin_minus_one), %g5 + ld [%g5 + %lo(nwin_minus_one)], %g5 + rdpr %cwp, %g6 + deccc %g6 + movneg %xcc, %g5, %g6 ! if (--%cwp < 0) %g6 = nwin-1 + wrpr %g6, %cwp + clr %l0 + clr %l1 + clr %l2 + clr %l3 + clr %l4 + clr %l5 + clr %l6 + clr %l7 + wrpr %g0, %g7, %wstate + saved + retry + + /* * User trap entry point. * ==== //depot/projects/kmacy_sun4v/src/sys/sun4v/sun4v/locore.S#4 (text+ko) ==== @@ -37,7 +37,10 @@ .globl kernbase .set kernbase,KERNBASE - + + .globl nwin_minus_one + .set nwin_minus_one,7 + /* * void _start(caddr_t metadata, u_long o1, u_long o2, u_long o3, * u_long ofw_vec)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200602180257.k1I2vb8m083029>