Date: Thu, 19 Mar 2009 14:11:03 +0000 (UTC) From: Marius Strobl <marius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-7@freebsd.org Subject: svn commit: r190051 - in stable/7/sys: . boot/sparc64/loader contrib/pf dev/ath/ath_hal dev/cxgb sparc64/include sparc64/sparc64 sun4v/include Message-ID: <200903191411.n2JEB3RD064233@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: marius Date: Thu Mar 19 14:11:03 2009 New Revision: 190051 URL: http://svn.freebsd.org/changeset/base/190051 Log: MFC: r182877, r188455, r188477 USIII and beyond CPUs have stricter requirements when it comes to synchronization needed after stores to internal ASIs in order to make side-effects visible. This mainly requires the MEMBAR #Sync after such stores to be replaced with a FLUSH. We use KERNBASE in the kernel and the newly introduced PROMBASE for the loader as the addresses to FLUSH as these are guaranteed to not trap. Actually, the USII synchronization rules also already require a FLUSH in pretty much all of the cases changed. We're also hitting an additional USIII synchronization rule which requires stores to AA_IMMU_SFSR to be immediately followed by a DONE, FLUSH or RETRY. Doing so triggers a RED state exception though so leave the MEMBAR #Sync. Linux apparently also has gotten away with doing the same for quite some time now, apart from the fact that it's not clear to me why we need to clear the valid bit from the SFSR in the first place. Reviewed by: nwhitehorn Modified: stable/7/sys/ (props changed) stable/7/sys/boot/sparc64/loader/main.c stable/7/sys/contrib/pf/ (props changed) stable/7/sys/dev/ath/ath_hal/ (props changed) stable/7/sys/dev/cxgb/ (props changed) stable/7/sys/sparc64/include/vmparam.h stable/7/sys/sparc64/sparc64/exception.S stable/7/sys/sparc64/sparc64/mp_exception.S stable/7/sys/sparc64/sparc64/pmap.c stable/7/sys/sparc64/sparc64/support.S stable/7/sys/sparc64/sparc64/swtch.S stable/7/sys/sparc64/sparc64/tlb.c stable/7/sys/sun4v/include/vmparam.h Modified: stable/7/sys/boot/sparc64/loader/main.c ============================================================================== --- stable/7/sys/boot/sparc64/loader/main.c Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/boot/sparc64/loader/main.c Thu Mar 19 14:11:03 2009 (r190051) @@ -37,6 +37,7 @@ __FBSDID("$FreeBSD$"); #include <machine/tlb.h> #include <machine/upa.h> #include <machine/ver.h> +#include <machine/vmparam.h> #include "bootstrap.h" #include "libofw.h" @@ -356,7 +357,7 @@ __elfN(exec)(struct preloaded_file *fp) return (error); printf("jumping to kernel entry at %#lx.\n", e->e_entry); -#if LOADER_DEBUG +#ifdef LOADER_DEBUG pmap_print_tlb_sun4u(); #endif @@ -461,7 +462,7 @@ itlb_enter_sun4u(u_long vpn, u_long data stxa(AA_IMMU_TAR, ASI_IMMU, TLB_TAR_VA(vpn) | TLB_TAR_CTX(TLB_CTX_KERNEL)); stxa(0, ASI_ITLB_DATA_IN_REG, data); - membar(Sync); + flush(PROMBASE); wrpr(pstate, reg, 0); } @@ -726,7 +727,7 @@ exit(int code) } #ifdef LOADER_DEBUG -static const char *page_sizes[] = { +static const char *const page_sizes[] = { " 8k", " 64k", "512k", " 4m" }; Modified: stable/7/sys/sparc64/include/vmparam.h ============================================================================== --- stable/7/sys/sparc64/include/vmparam.h Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/include/vmparam.h Thu Mar 19 14:11:03 2009 (r190051) @@ -40,7 +40,6 @@ * $FreeBSD$ */ - #ifndef _MACHINE_VMPARAM_H_ #define _MACHINE_VMPARAM_H_ @@ -203,6 +202,7 @@ #define VM_MAXUSER_ADDRESS (VM_MAX_USER_ADDRESS) #define KERNBASE (VM_MIN_KERNEL_ADDRESS) +#define PROMBASE (VM_MIN_PROM_ADDRESS) #define USRSTACK (VM_MAX_USER_ADDRESS) /* Modified: stable/7/sys/sparc64/sparc64/exception.S ============================================================================== --- stable/7/sys/sparc64/sparc64/exception.S Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/sparc64/exception.S Thu Mar 19 14:11:03 2009 (r190051) @@ -498,6 +498,11 @@ END(rsf_fatal) wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 + /* + * XXX in theory, a store to AA_IMMU_SFSR must be immediately + * followed by a DONE, FLUSH or RETRY for USIII. In practice, + * this triggers a RED state exception though. + */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl0_sfsr_trap @@ -716,8 +721,9 @@ ENTRY(tl0_immu_miss_trap) * Put back the contents of the tag access register, in case we * faulted. */ + sethi %hi(KERNBASE), %g2 stxa %g1, [%g0 + AA_IMMU_TAR] %asi - membar #Sync + flush %g2 /* * Switch to alternate globals. @@ -1213,6 +1219,11 @@ END(tl0_fp_restore) wr %g0, ASI_IMMU, %asi rdpr %tpc, %g3 ldxa [%g0 + AA_IMMU_SFSR] %asi, %g4 + /* + * XXX in theory, a store to AA_IMMU_SFSR must be immediately + * followed by a DONE, FLUSH or RETRY for USIII. In practice, + * this triggers a RED state exception though. + */ stxa %g0, [%g0 + AA_IMMU_SFSR] %asi membar #Sync ba %xcc, tl1_insn_exceptn_trap Modified: stable/7/sys/sparc64/sparc64/mp_exception.S ============================================================================== --- stable/7/sys/sparc64/sparc64/mp_exception.S Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/sparc64/mp_exception.S Thu Mar 19 14:11:03 2009 (r190051) @@ -199,9 +199,10 @@ ENTRY(tl_ipi_tlb_page_demap) ldx [%g5 + ITA_VA], %g2 or %g2, %g3, %g2 + sethi %hi(KERNBASE), %g3 stxa %g0, [%g2] ASI_DMMU_DEMAP stxa %g0, [%g2] ASI_IMMU_DEMAP - membar #Sync + flush %g3 IPI_DONE(%g5, %g1, %g2, %g3) retry @@ -234,13 +235,13 @@ ENTRY(tl_ipi_tlb_range_demap) ldx [%g5 + ITA_START], %g1 ldx [%g5 + ITA_END], %g2 - set PAGE_SIZE, %g6 - 1: or %g1, %g3, %g4 + sethi %hi(KERNBASE), %g6 stxa %g0, [%g4] ASI_DMMU_DEMAP stxa %g0, [%g4] ASI_IMMU_DEMAP - membar #Sync + flush %g6 + set PAGE_SIZE, %g6 add %g1, %g6, %g1 cmp %g1, %g2 blt,a,pt %xcc, 1b @@ -265,9 +266,10 @@ ENTRY(tl_ipi_tlb_context_demap) #endif mov TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, %g1 + sethi %hi(KERNBASE), %g3 stxa %g0, [%g1] ASI_DMMU_DEMAP stxa %g0, [%g1] ASI_IMMU_DEMAP - membar #Sync + flush %g3 IPI_DONE(%g5, %g1, %g2, %g3) retry Modified: stable/7/sys/sparc64/sparc64/pmap.c ============================================================================== --- stable/7/sys/sparc64/sparc64/pmap.c Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/sparc64/pmap.c Thu Mar 19 14:11:03 2009 (r190051) @@ -556,7 +556,7 @@ pmap_map_tsb(void) * FP block operations in the kernel). */ stxa(AA_DMMU_SCXR, ASI_DMMU, TLB_CTX_KERNEL); - membar(Sync); + flush(KERNBASE); intr_restore(s); } @@ -1961,7 +1961,7 @@ pmap_activate(struct thread *td) stxa(AA_DMMU_TSB, ASI_DMMU, pm->pm_tsb); stxa(AA_IMMU_TSB, ASI_IMMU, pm->pm_tsb); stxa(AA_DMMU_PCXR, ASI_DMMU, context); - membar(Sync); + flush(KERNBASE); mtx_unlock_spin(&sched_lock); } Modified: stable/7/sys/sparc64/sparc64/support.S ============================================================================== --- stable/7/sys/sparc64/sparc64/support.S Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/sparc64/support.S Thu Mar 19 14:11:03 2009 (r190051) @@ -780,8 +780,9 @@ ENTRY(openfirmware_exit) sub %l0, SPOFF, %fp ! setup a stack in a locked page sub %l0, SPOFF + CCFSZ, %sp mov AA_DMMU_PCXR, %l3 ! force primary DMMU context 0 + sethi %hi(KERNBASE), %l5 stxa %g0, [%l3] ASI_DMMU - membar #Sync + flush %l5 wrpr %g0, 0, %tl ! force trap level 0 call %l6 mov %i0, %o0 Modified: stable/7/sys/sparc64/sparc64/swtch.S ============================================================================== --- stable/7/sys/sparc64/sparc64/swtch.S Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/sparc64/swtch.S Thu Mar 19 14:11:03 2009 (r190051) @@ -237,8 +237,9 @@ ENTRY(cpu_switch) mov AA_IMMU_TSB, %i5 stxa %i4, [%i5] ASI_IMMU mov AA_DMMU_PCXR, %i5 + sethi %hi(KERNBASE), %i4 stxa %i3, [%i5] ASI_DMMU - membar #Sync + flush %i4 /* * Done, return and load the new process's window from the stack. Modified: stable/7/sys/sparc64/sparc64/tlb.c ============================================================================== --- stable/7/sys/sparc64/sparc64/tlb.c Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sparc64/sparc64/tlb.c Thu Mar 19 14:11:03 2009 (r190051) @@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include <machine/pmap.h> #include <machine/smp.h> #include <machine/tlb.h> +#include <machine/vmparam.h> PMAP_STATS_VAR(tlb_ncontext_demap); PMAP_STATS_VAR(tlb_npage_demap); @@ -85,7 +86,7 @@ tlb_context_demap(struct pmap *pm) s = intr_disable(); stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_PRIMARY | TLB_DEMAP_CONTEXT, ASI_IMMU_DEMAP, 0); - membar(Sync); + flush(KERNBASE); intr_restore(s); } ipi_wait(cookie); @@ -111,7 +112,7 @@ tlb_page_demap(struct pmap *pm, vm_offse s = intr_disable(); stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0); - membar(Sync); + flush(KERNBASE); intr_restore(s); } ipi_wait(cookie); @@ -139,7 +140,7 @@ tlb_range_demap(struct pmap *pm, vm_offs for (va = start; va < end; va += PAGE_SIZE) { stxa(TLB_DEMAP_VA(va) | flags, ASI_DMMU_DEMAP, 0); stxa(TLB_DEMAP_VA(va) | flags, ASI_IMMU_DEMAP, 0); - membar(Sync); + flush(KERNBASE); } intr_restore(s); } Modified: stable/7/sys/sun4v/include/vmparam.h ============================================================================== --- stable/7/sys/sun4v/include/vmparam.h Thu Mar 19 14:09:20 2009 (r190050) +++ stable/7/sys/sun4v/include/vmparam.h Thu Mar 19 14:11:03 2009 (r190051) @@ -40,7 +40,6 @@ * $FreeBSD$ */ - #ifndef _MACHINE_VMPARAM_H_ #define _MACHINE_VMPARAM_H_ @@ -203,6 +202,7 @@ #define VM_MAXUSER_ADDRESS (VM_MAX_USER_ADDRESS) #define KERNBASE (VM_MIN_KERNEL_ADDRESS) +#define PROMBASE (VM_MIN_PROM_ADDRESS) #define USRSTACK (VM_MAX_USER_ADDRESS) /*
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200903191411.n2JEB3RD064233>