Date: Sun, 19 May 2002 18:21:45 -0700 (PDT) From: Jake Burkholder <jake@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 11575 for review Message-ID: <200205200121.g4K1Ljs65264@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://people.freebsd.org/~peter/p4db/chv.cgi?CH=11575 Change 11575 by jake@jake_sparc64 on 2002/05/19 18:20:46 Add SMP aware cache primitives for invalidating a single physical page from the icache and/or dcache. Hopefully this will improve stability for SMP. Affected files ... ... //depot/projects/sparc64/sys/sparc64/include/cache.h#12 edit ... //depot/projects/sparc64/sys/sparc64/include/smp.h#15 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/cache.c#18 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/genassym.c#68 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/machdep.c#79 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/mp_exception.s#9 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/mp_machdep.c#15 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/pmap.c#86 edit ... //depot/projects/sparc64/sys/sparc64/sparc64/vm_machdep.c#54 edit Differences ... ==== //depot/projects/sparc64/sys/sparc64/include/cache.h#12 (text+ko) ==== @@ -103,25 +103,48 @@ void ecache_inval_phys(vm_offset_t, vm_offset_t); #endif +void dcache_page_inval(vm_offset_t pa); +void icache_page_inval(vm_offset_t pa); + +#define DC_TAG_SHIFT 2 +#define DC_VALID_SHIFT 0 + +#define DC_TAG_BITS 28 +#define DC_VALID_BITS 2 + +#define DC_TAG_MASK ((1 << DC_TAG_BITS) - 1) +#define DC_VALID_MASK ((1 << DC_VALID_BITS) - 1) + +#define IC_TAG_SHIFT 7 +#define IC_VALID_SHIFT 36 + +#define IC_TAG_BITS 28 +#define IC_VALID_BITS 1 + +#define IC_TAG_MASK ((1 << IC_TAG_BITS) - 1) +#define IC_VALID_MASK ((1 << IC_VALID_BITS) - 1) + /* * Cache control information. */ struct cacheinfo { - int c_enabled; /* true => cache is enabled */ - int ic_size; /* instruction cache */ - int ic_set; - int ic_l2set; - int ic_assoc; - int ic_linesize; - int dc_size; /* data cache */ - int dc_l2size; - int dc_assoc; - int dc_linesize; - int ec_size; /* external cache info */ - int ec_assoc; - int ec_l2set; - int ec_linesize; - int ec_l2linesize; + u_int c_enabled; /* true => cache is enabled */ + u_int ic_size; /* instruction cache */ + u_int ic_set; + u_int ic_l2set; + u_int ic_assoc; + u_int ic_linesize; + u_int dc_size; /* data cache */ + u_int dc_l2size; + u_int dc_assoc; + u_int dc_linesize; + u_int ec_size; /* external cache info */ + u_int ec_assoc; + u_int ec_l2set; + u_int ec_linesize; + u_int ec_l2linesize; }; +extern struct cacheinfo cache; + #endif /* !_MACHINE_CACHE_H_ */ ==== //depot/projects/sparc64/sys/sparc64/include/smp.h#15 (text+ko) ==== @@ -57,6 +57,11 @@ struct tte csa_ttes[PCPU_PAGES]; }; +struct ipi_cache_args { + u_int ica_mask; + u_long ica_pa; +}; + struct ipi_tlb_args { u_int ita_mask; u_long ita_tlb; @@ -80,6 +85,7 @@ vm_offset_t mp_tramp_alloc(void); +extern struct ipi_cache_args ipi_cache_args; extern struct ipi_level_args ipi_level_args; extern struct ipi_tlb_args ipi_tlb_args; @@ -91,6 +97,8 @@ extern void mp_startup(void); +extern char tl_ipi_dcache_page_inval[]; +extern char tl_ipi_icache_page_inval[]; extern char tl_ipi_level[]; extern char tl_ipi_test[]; extern char tl_ipi_tlb_context_demap[]; @@ -99,6 +107,36 @@ #ifdef SMP +static __inline void * +ipi_dcache_page_inval(vm_offset_t pa) +{ + struct ipi_cache_args *ica; + + if (smp_cpus == 1) + return (NULL); + ica = &ipi_cache_args; + ica->ica_mask = all_cpus; + ica->ica_pa = pa; + cpu_ipi_selected(all_cpus, 0, (u_long)tl_ipi_dcache_page_inval, + (u_long)ica); + return (&ica->ica_mask); +} + +static __inline void * +ipi_icache_page_inval(vm_offset_t pa) +{ + struct ipi_cache_args *ica; + + if (smp_cpus == 1) + return (NULL); + ica = &ipi_cache_args; + ica->ica_mask = all_cpus; + ica->ica_pa = pa; + cpu_ipi_selected(all_cpus, 0, (u_long)tl_ipi_icache_page_inval, + (u_long)ica); + return (&ica->ica_mask); +} + #ifdef _MACHINE_PMAP_H_ static __inline void * @@ -174,6 +212,18 @@ #else static __inline void * +ipi_dcache_page_inval(vm_offset_t pa) +{ + return (NULL); +} + +static __inline void * +ipi_icache_page_inval(vm_offset_t pa) +{ + return (NULL); +} + +static __inline void * ipi_tlb_context_demap(struct pmap *pm) { return (NULL); ==== //depot/projects/sparc64/sys/sparc64/sparc64/cache.c#18 (text+ko) ==== @@ -152,6 +152,7 @@ #include <sys/param.h> #include <sys/proc.h> +#include <sys/smp.h> #include <sys/systm.h> #include <vm/vm.h> @@ -163,12 +164,12 @@ #include <machine/fsr.h> #include <machine/pcb.h> #include <machine/pmap.h> +#include <machine/smp.h> #include <machine/tte.h> #include <machine/ver.h> #include <machine/vmparam.h> -static struct cacheinfo cache; -extern vm_offset_t cache_tmp_va; +struct cacheinfo cache; /* Read to %g0, needed for E$ access. */ #define CDIAG_RDG0(asi, addr) \ @@ -215,6 +216,64 @@ cache.c_enabled = 1; /* enable cache flushing */ } +void +dcache_page_inval(vm_offset_t pa) +{ + u_long target; + void *cookie; + u_long addr; + u_long tag; + + KASSERT((pa & PAGE_MASK) == 0, + ("dcache_page_inval: pa not page aligned")); + + if (!cache.c_enabled) + return; + target = pa >> (PAGE_SHIFT - DC_TAG_SHIFT); + critical_enter(); + cookie = ipi_dcache_page_inval(pa); + for (addr = 0; addr < cache.dc_size; addr += cache.dc_linesize) { + tag = ldxa(addr, ASI_DCACHE_TAG); + if (((tag >> DC_VALID_SHIFT) & DC_VALID_MASK) == 0) + continue; + tag &= DC_TAG_MASK << DC_TAG_SHIFT; + if (tag == target) + stxa_sync(addr, ASI_DCACHE_TAG, tag); + } + ipi_wait(cookie); + critical_exit(); +} + +void +icache_page_inval(vm_offset_t pa) +{ + register u_long tag __asm("%g1"); + u_long target; + void *cookie; + u_long addr; + + KASSERT((pa & PAGE_MASK) == 0, + ("icache_page_inval: pa not page aligned")); + + if (!cache.c_enabled) + return; + target = pa >> (PAGE_SHIFT - IC_TAG_SHIFT); + critical_enter(); + cookie = ipi_icache_page_inval(pa); + for (addr = 0; addr < cache.ic_size; addr += cache.ic_linesize) { + __asm __volatile("ldda [%1] %2, %%g0" /*, %g1 */ + : "=r" (tag) : "r" (addr), "n" (ASI_ICACHE_TAG)); + if (((tag >> IC_VALID_SHIFT) & IC_VALID_MASK) == 0) + continue; + tag &= IC_TAG_MASK << IC_TAG_SHIFT; + if (tag == target) + stxa_sync(addr, ASI_ICACHE_TAG, tag); + } + ipi_wait(cookie); + critical_exit(); +} + + /* Flush a range of addresses from I$ using the flush instruction. */ void icache_flush(vm_offset_t start, vm_offset_t end) ==== //depot/projects/sparc64/sys/sparc64/sparc64/genassym.c#68 (text+ko) ==== @@ -63,6 +63,7 @@ #include <machine/setjmp.h> #include <machine/sigframe.h> #include <machine/smp.h> +#include <machine/cache.h> #include <machine/pv.h> #include <machine/tte.h> #include <machine/tlb.h> @@ -120,6 +121,22 @@ ASSYM(CSA_VER, offsetof(struct cpu_start_args, csa_ver)); ASSYM(CSA_TTES, offsetof(struct cpu_start_args, csa_ttes)); +ASSYM(DC_TAG_SHIFT, DC_TAG_SHIFT); +ASSYM(DC_TAG_MASK, DC_TAG_MASK); +ASSYM(DC_VALID_SHIFT, DC_VALID_SHIFT); +ASSYM(DC_VALID_MASK, DC_VALID_MASK); +ASSYM(IC_TAG_SHIFT, IC_TAG_SHIFT); +ASSYM(IC_TAG_MASK, IC_TAG_MASK); +ASSYM(IC_VALID_SHIFT, IC_VALID_SHIFT); +ASSYM(IC_VALID_MASK, IC_VALID_MASK); + +ASSYM(DC_SIZE, offsetof(struct cacheinfo, dc_size)); +ASSYM(DC_LINESIZE, offsetof(struct cacheinfo, dc_linesize)); +ASSYM(IC_SIZE, offsetof(struct cacheinfo, ic_size)); +ASSYM(IC_LINESIZE, offsetof(struct cacheinfo, ic_linesize)); + +ASSYM(ICA_PA, offsetof(struct ipi_cache_args, ica_pa)); + ASSYM(KTR_PROC, KTR_PROC); ASSYM(KTR_TRAP, KTR_TRAP); ASSYM(KTR_SMP, KTR_SMP); ==== //depot/projects/sparc64/sys/sparc64/sparc64/machdep.c#79 (text+ko) ==== @@ -86,7 +86,6 @@ #include <ddb/ddb.h> -#include <machine/cache.h> #include <machine/clock.h> #include <machine/cpu.h> #include <machine/fp.h> @@ -95,6 +94,7 @@ #include <machine/metadata.h> #include <machine/ofw_machdep.h> #include <machine/smp.h> +#include <machine/cache.h> #include <machine/pmap.h> #include <machine/pstate.h> #include <machine/reg.h> ==== //depot/projects/sparc64/sys/sparc64/sparc64/mp_exception.s#9 (text+ko) ==== @@ -44,6 +44,86 @@ nop /* + * Invalidate a phsyical page in the data cache. + */ +ENTRY(tl_ipi_dcache_page_inval) +#if KTR_COMPILE & KTR_SMP + CATR(KTR_SMP, "ipi_dcache_page_inval: pa=%#lx" + , %g1, %g2, %g3, 7, 8, 9) + ldx [%g5 + ICA_PA], %g2 + stx %g2, [%g1 + KTR_PARM1] +9: +#endif + + ldx [%g5 + ICA_PA], %g6 + srlx %g6, PAGE_SHIFT - DC_TAG_SHIFT, %g6 + + SET(cache, %g3, %g2) + lduw [%g2 + DC_SIZE], %g3 + lduw [%g2 + DC_LINESIZE], %g4 + sub %g3, %g4, %g2 + +1: ldxa [%g2] ASI_DCACHE_TAG, %g1 + srlx %g1, DC_VALID_SHIFT, %g3 + andcc %g3, DC_VALID_MASK, %g0 + bz,pt %xcc, 2f + set DC_TAG_MASK, %g3 + sllx %g3, DC_TAG_SHIFT, %g3 + and %g1, %g3, %g1 + cmp %g1, %g6 + bne,a,pt %xcc, 2f + nop + stxa %g1, [%g2] ASI_DCACHE_TAG + membar #Sync + +2: brgz,pt %g2, 1b + sub %g2, %g4, %g2 + + IPI_WAIT(%g5, %g1, %g2, %g3) + retry +END(tl_ipi_dcache_page_inval) + +/* + * Invalidate a phsyical page in the instruction cache. + */ +ENTRY(tl_ipi_icache_page_inval) +#if KTR_COMPILE & KTR_SMP + CATR(KTR_SMP, "ipi_icache_page_inval: pa=%#lx" + , %g1, %g2, %g3, 7, 8, 9) + ldx [%g5 + ICA_PA], %g2 + stx %g2, [%g1 + KTR_PARM1] +9: +#endif + + ldx [%g5 + ICA_PA], %g6 + srlx %g6, PAGE_SHIFT - IC_TAG_SHIFT, %g6 + + SET(cache, %g3, %g2) + lduw [%g2 + IC_SIZE], %g3 + lduw [%g2 + IC_LINESIZE], %g4 + sub %g3, %g4, %g2 + +1: ldda [%g2] ASI_ICACHE_TAG, %g0 /*, %g1 */ + srlx %g1, IC_VALID_SHIFT, %g3 + andcc %g3, IC_VALID_MASK, %g0 + bz,pt %xcc, 2f + set IC_TAG_MASK, %g3 + sllx %g3, IC_TAG_SHIFT, %g3 + and %g1, %g3, %g1 + cmp %g1, %g6 + bne,a,pt %xcc, 2f + nop + stxa %g1, [%g2] ASI_ICACHE_TAG + membar #Sync + +2: brgz,pt %g2, 1b + sub %g2, %g4, %g2 + + IPI_WAIT(%g5, %g1, %g2, %g3) + retry +END(tl_ipi_icache_page_inval) + +/* * Trigger a softint at the desired level. */ ENTRY(tl_ipi_level) ==== //depot/projects/sparc64/sys/sparc64/sparc64/mp_machdep.c#15 (text+ko) ==== @@ -95,6 +95,7 @@ * kernel. */ struct cpu_start_args cpu_start_args = { 0, -1, -1, 0, 0 }; +struct ipi_cache_args ipi_cache_args; struct ipi_tlb_args ipi_tlb_args; vm_offset_t mp_tramp; ==== //depot/projects/sparc64/sys/sparc64/sparc64/pmap.c#86 (text+ko) ==== @@ -658,7 +658,7 @@ if (m->md.vcolor != col) { /* XXXSMP */ PMAP_STATS_INC(pmap_nvdflush); - dcache_inval_phys(pa, pa + PAGE_SIZE - 1); + dcache_page_inval(pa); } m->md.vcolor = col; m->md.flags &= ~MPF_VIRGIN; @@ -687,7 +687,7 @@ } } /* XXXSMP */ - dcache_inval_phys(pa, pa + PAGE_SIZE - 1); + dcache_page_inval(pa); return (0); } @@ -1542,7 +1542,7 @@ PMAP_STATS_INC(pmap_niflush); if (pm == kernel_pmap) PMAP_STATS_INC(pmap_niflush_k); - icache_inval_phys(pa, pa + PAGE_SIZE - 1); + icache_page_inval(pa); m->md.flags |= MPF_IFLUSHED; } } @@ -1655,7 +1655,7 @@ vm_offset_t pa = VM_PAGE_TO_PHYS(m); CTR1(KTR_PMAP, "pmap_zero_page: pa=%#lx", pa); - dcache_inval_phys(pa, pa + PAGE_SIZE - 1); + dcache_page_inval(pa); aszero(ASI_PHYS_USE_EC, pa, PAGE_SIZE); } @@ -1667,7 +1667,7 @@ CTR3(KTR_PMAP, "pmap_zero_page_area: pa=%#lx off=%#x size=%#x", pa, off, size); KASSERT(off + size <= PAGE_SIZE, ("pmap_zero_page_area: bad off/size")); - dcache_inval_phys(pa + off, pa + off + size - 1); + dcache_page_inval(pa); aszero(ASI_PHYS_USE_EC, pa + off, size); } @@ -1681,7 +1681,7 @@ vm_offset_t dst = VM_PAGE_TO_PHYS(mdst); CTR2(KTR_PMAP, "pmap_copy_page: src=%#lx dst=%#lx", src, dst); - dcache_inval_phys(dst, dst + PAGE_SIZE - 1); + dcache_page_inval(dst); ascopy(ASI_PHYS_USE_EC, src, dst, PAGE_SIZE); } @@ -1776,8 +1776,7 @@ if ((tp->tte_data & TD_EXEC) != 0) { PMAP_STATS_INC(pmap_niflush); PMAP_STATS_INC(pmap_pgval_niflush); - icache_inval_phys(VM_PAGE_TO_PHYS(m), - VM_PAGE_TO_PHYS(m) + PAGE_SIZE - 1); + icache_page_inval(VM_PAGE_TO_PHYS(m)); m->md.flags |= MPF_IFLUSHED; break; } ==== //depot/projects/sparc64/sys/sparc64/sparc64/vm_machdep.c#54 (text+ko) ==== @@ -507,7 +507,7 @@ VMMD_STATS_INC(niorncflush); #endif /* XXX: use dcache_inval()? */ - dcache_inval_phys(pa, pa + PAGE_SIZE - 1); + dcache_page_inval(pa); } } break; @@ -559,8 +559,7 @@ if ((m[i]->md.flags & MPF_NC) != 0) VMMD_STATS_INC(niowncflush); #endif - dcache_inval_phys(pa, - pa + PAGE_SIZE - 1); + dcache_page_inval(pa); m[i]->md.flags |= MPF_VMIO_DONE; } /* @@ -577,8 +576,7 @@ */ VMMD_STATS_INC(niowflush); VMMD_STATS_INC(niowmflush); - dcache_inval_phys(pa, - pa + PAGE_SIZE - 1); + dcache_page_inval(pa); m[i]->md.vcolor = -1; m[i]->md.flags = MPF_VMIO_DONE; } else if ((m[i]->md.flags & MPF_VMIO_DONE) == 0) { To Unsubscribe: send mail to majordomo@FreeBSD.org with "unsubscribe p4-projects" in the body of the message
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200205200121.g4K1Ljs65264>