Date: Tue, 26 Jan 2010 20:58:09 +0000 (UTC) From: John Baldwin <jhb@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-7@freebsd.org Subject: svn commit: r203047 - in stable/7/sys: amd64/amd64 amd64/include i386/i386 i386/include Message-ID: <201001262058.o0QKw9T6002049@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jhb Date: Tue Jan 26 20:58:09 2010 New Revision: 203047 URL: http://svn.freebsd.org/changeset/base/203047 Log: MFC 195820, 195940, 196643, 197663, 199067, 199215, and 199253: Optimize the cache flushing done when changing caching attributes of pages by doing nothing for CPUs that support self-snooping and using CLFLUSH instead of a full cache invalidate when possible. - On i386 take care of possible mappings of the page by sf buffer by utilizing the mapping for clflush, otherwise map the page transiently. Amd64 uses the direct map. - Do not use CLFLUSH on Intel CPUs due to problems with flushing the local APIC range by default. This can be further controlled via the hw.clflush_disable loader tunable. A setting of 1 disables the use of CLFLUSH. A setting of 0 allows CLFLUSH to be used for Intel CPUs when CPUID_SS is not present. Approved by: re (kib) Modified: stable/7/sys/amd64/amd64/initcpu.c stable/7/sys/amd64/amd64/machdep.c stable/7/sys/amd64/amd64/pmap.c stable/7/sys/amd64/include/cpufunc.h stable/7/sys/amd64/include/md_var.h stable/7/sys/i386/i386/initcpu.c stable/7/sys/i386/i386/pmap.c stable/7/sys/i386/i386/vm_machdep.c stable/7/sys/i386/include/cpufunc.h stable/7/sys/i386/include/md_var.h stable/7/sys/i386/include/pmap.h stable/7/sys/i386/include/sf_buf.h Directory Properties: stable/7/sys/ (props changed) stable/7/sys/cddl/contrib/opensolaris/ (props changed) stable/7/sys/contrib/dev/acpica/ (props changed) stable/7/sys/contrib/pf/ (props changed) Modified: stable/7/sys/amd64/amd64/initcpu.c ============================================================================== --- stable/7/sys/amd64/amd64/initcpu.c Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/amd64/amd64/initcpu.c Tue Jan 26 20:58:09 2010 (r203047) @@ -47,6 +47,12 @@ __FBSDID("$FreeBSD$"); static int hw_instruction_sse; SYSCTL_INT(_hw, OID_AUTO, instruction_sse, CTLFLAG_RD, &hw_instruction_sse, 0, "SIMD/MMX2 instructions available in CPU"); +/* + * -1: automatic (default) + * 0: keep enable CLFLUSH + * 1: force disable CLFLUSH + */ +static int hw_clflush_disable = -1; int cpu; /* Are we 386, 386sx, 486, etc? */ u_int cpu_feature; /* Feature flags */ @@ -65,6 +71,7 @@ char cpu_vendor[20]; /* CPU Origin code u_int cpu_vendor_id; /* CPU vendor ID */ u_int cpu_fxsr; /* SSE enabled */ u_int cpu_mxcsr_mask; /* Valid bits in mxcsr */ +u_int cpu_clflush_line_size = 32; SYSCTL_UINT(_hw, OID_AUTO, via_feature_rng, CTLFLAG_RD, &via_feature_rng, 0, "VIA C3/C7 RNG feature available in CPU"); @@ -157,3 +164,31 @@ initializecpu(void) CPUID_TO_MODEL(cpu_id) >= 0xf) init_via(); } + +void +initializecpucache() +{ + + /* + * CPUID with %eax = 1, %ebx returns + * Bits 15-8: CLFLUSH line size + * (Value * 8 = cache line size in bytes) + */ + if ((cpu_feature & CPUID_CLFSH) != 0) + cpu_clflush_line_size = ((cpu_procinfo >> 8) & 0xff) * 8; + /* + * XXXKIB: (temporary) hack to work around traps generated when + * CLFLUSHing APIC registers window. + */ + TUNABLE_INT_FETCH("hw.clflush_disable", &hw_clflush_disable); + if (cpu_vendor_id == CPU_VENDOR_INTEL && !(cpu_feature & CPUID_SS) && + hw_clflush_disable == -1) + cpu_feature &= ~CPUID_CLFSH; + /* + * Allow to disable CLFLUSH feature manually by + * hw.clflush_disable tunable. This may help Xen guest on some AMD + * CPUs. + */ + if (hw_clflush_disable == 1) + cpu_feature &= ~CPUID_CLFSH; +} Modified: stable/7/sys/amd64/amd64/machdep.c ============================================================================== --- stable/7/sys/amd64/amd64/machdep.c Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/amd64/amd64/machdep.c Tue Jan 26 20:58:09 2010 (r203047) @@ -1332,6 +1332,7 @@ hammer_time(u_int64_t modulep, u_int64_t identify_cpu(); /* Final stage of CPU initialization */ initializecpu(); /* Initialize CPU registers */ + initializecpucache(); /* make an initial tss so cpu can get interrupt stack on syscall! */ common_tss[0].tss_rsp0 = thread0.td_kstack + \ Modified: stable/7/sys/amd64/amd64/pmap.c ============================================================================== --- stable/7/sys/amd64/amd64/pmap.c Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/amd64/amd64/pmap.c Tue Jan 26 20:58:09 2010 (r203047) @@ -228,6 +228,7 @@ static vm_page_t pmap_enter_quick_locked vm_page_t m, vm_prot_t prot, vm_page_t mpte); static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); +static void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva); static boolean_t pmap_is_modified_pvh(struct md_page *pvh); static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va); @@ -934,6 +935,40 @@ pmap_invalidate_cache(void) } #endif /* !SMP */ +static void +pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) +{ + + KASSERT((sva & PAGE_MASK) == 0, + ("pmap_invalidate_cache_range: sva not page-aligned")); + KASSERT((eva & PAGE_MASK) == 0, + ("pmap_invalidate_cache_range: eva not page-aligned")); + + if (cpu_feature & CPUID_SS) + ; /* If "Self Snoop" is supported, do nothing. */ + else if (cpu_feature & CPUID_CLFSH) { + + /* + * Otherwise, do per-cache line flush. Use the mfence + * instruction to insure that previous stores are + * included in the write-back. The processor + * propagates flush to other processors in the cache + * coherence domain. + */ + mfence(); + for (; sva < eva; sva += cpu_clflush_line_size) + clflush(sva); + mfence(); + } else { + + /* + * No targeted cache flush methods are supported by CPU, + * globally invalidate cache as a last resort. + */ + pmap_invalidate_cache(); + } +} + /* * Are we current address space or kernel? */ @@ -4229,7 +4264,8 @@ pmap_pde_attr(pd_entry_t *pde, int cache void * pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) { - vm_offset_t va, tmpva, offset; + vm_offset_t va, offset; + vm_size_t tmpsize; /* * If the specified range of physical addresses fits within the direct @@ -4246,14 +4282,10 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_ if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); pa = trunc_page(pa); - for (tmpva = va; size > 0; ) { - pmap_kenter_attr(tmpva, pa, mode); - size -= PAGE_SIZE; - tmpva += PAGE_SIZE; - pa += PAGE_SIZE; - } - pmap_invalidate_range(kernel_pmap, va, tmpva); - pmap_invalidate_cache(); + for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) + pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); + pmap_invalidate_range(kernel_pmap, va, va + tmpsize); + pmap_invalidate_cache_range(va, va + tmpsize); return ((void *)(va + offset)); } @@ -4473,7 +4505,7 @@ pmap_change_attr_locked(vm_offset_t va, */ if (changed) { pmap_invalidate_range(kernel_pmap, base, tmpva); - pmap_invalidate_cache(); + pmap_invalidate_cache_range(base, tmpva); } return (error); } Modified: stable/7/sys/amd64/include/cpufunc.h ============================================================================== --- stable/7/sys/amd64/include/cpufunc.h Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/amd64/include/cpufunc.h Tue Jan 26 20:58:09 2010 (r203047) @@ -100,6 +100,13 @@ bsrq(u_long mask) } static __inline void +clflush(u_long addr) +{ + + __asm __volatile("clflush %0" : : "m" (*(char *)addr)); +} + +static __inline void disable_intr(void) { __asm __volatile("cli" : : : "memory"); @@ -338,6 +345,13 @@ outw(u_int port, u_short data) } static __inline void +mfence(void) +{ + + __asm__ __volatile("mfence" : : : "memory"); +} + +static __inline void ia32_pause(void) { __asm __volatile("pause"); Modified: stable/7/sys/amd64/include/md_var.h ============================================================================== --- stable/7/sys/amd64/include/md_var.h Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/amd64/include/md_var.h Tue Jan 26 20:58:09 2010 (r203047) @@ -47,6 +47,7 @@ extern u_int amd_feature2; extern u_int amd_pminfo; extern u_int via_feature_rng; extern u_int via_feature_xcrypt; +extern u_int cpu_clflush_line_size; extern u_int cpu_fxsr; extern u_int cpu_high; extern u_int cpu_id; @@ -75,6 +76,7 @@ void doreti_iret_fault(void) __asm(__STR void dump_add_page(vm_paddr_t); void dump_drop_page(vm_paddr_t); void initializecpu(void); +void initializecpucache(void); void fillw(int /*u_short*/ pat, void *base, size_t cnt); void fpstate_drop(struct thread *td); int is_physical_memory(vm_paddr_t addr); Modified: stable/7/sys/i386/i386/initcpu.c ============================================================================== --- stable/7/sys/i386/i386/initcpu.c Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/i386/initcpu.c Tue Jan 26 20:58:09 2010 (r203047) @@ -75,6 +75,12 @@ static void init_mendocino(void); static int hw_instruction_sse; SYSCTL_INT(_hw, OID_AUTO, instruction_sse, CTLFLAG_RD, &hw_instruction_sse, 0, "SIMD/MMX2 instructions available in CPU"); +/* + * -1: automatic (default) + * 0: keep enable CLFLUSH + * 1: force disable CLFLUSH + */ +static int hw_clflush_disable = -1; /* Must *NOT* be BSS or locore will bzero these after setting them */ int cpu = 0; /* Are we 386, 386sx, 486, etc? */ @@ -91,6 +97,7 @@ u_int cpu_procinfo = 0; /* HyperThreadin u_int cpu_procinfo2 = 0; /* Multicore info */ char cpu_vendor[20] = ""; /* CPU Origin code */ u_int cpu_vendor_id = 0; /* CPU vendor ID */ +u_int cpu_clflush_line_size = 32; SYSCTL_UINT(_hw, OID_AUTO, via_feature_rng, CTLFLAG_RD, &via_feature_rng, 0, "VIA C3/C7 RNG feature available in CPU"); @@ -709,6 +716,29 @@ initializecpu(void) } enable_sse(); + /* + * CPUID with %eax = 1, %ebx returns + * Bits 15-8: CLFLUSH line size + * (Value * 8 = cache line size in bytes) + */ + if ((cpu_feature & CPUID_CLFSH) != 0) + cpu_clflush_line_size = ((cpu_procinfo >> 8) & 0xff) * 8; + /* + * XXXKIB: (temporary) hack to work around traps generated when + * CLFLUSHing APIC registers window. + */ + TUNABLE_INT_FETCH("hw.clflush_disable", &hw_clflush_disable); + if (cpu_vendor_id == CPU_VENDOR_INTEL && !(cpu_feature & CPUID_SS) && + hw_clflush_disable == -1) + cpu_feature &= ~CPUID_CLFSH; + /* + * Allow to disable CLFLUSH feature manually by + * hw.clflush_disable tunable. This may help Xen guest on some AMD + * CPUs. + */ + if (hw_clflush_disable == 1) + cpu_feature &= ~CPUID_CLFSH; + #if defined(PC98) && !defined(CPU_UPGRADE_HW_CACHE) /* * OS should flush L1 cache by itself because no PC-98 supports Modified: stable/7/sys/i386/i386/pmap.c ============================================================================== --- stable/7/sys/i386/i386/pmap.c Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/i386/pmap.c Tue Jan 26 20:58:09 2010 (r203047) @@ -119,6 +119,7 @@ __FBSDID("$FreeBSD$"); #include <sys/msgbuf.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/sf_buf.h> #include <sys/sx.h> #include <sys/vmmeter.h> #include <sys/sched.h> @@ -735,7 +736,7 @@ SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, pro * Determine the appropriate bits to set in a PTE or PDE for a specified * caching mode. */ -static int +int pmap_cache_bits(int mode, boolean_t is_pde) { int pat_flag, pat_index, cache_bits; @@ -944,6 +945,40 @@ pmap_invalidate_cache(void) } #endif /* !SMP */ +void +pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) +{ + + KASSERT((sva & PAGE_MASK) == 0, + ("pmap_invalidate_cache_range: sva not page-aligned")); + KASSERT((eva & PAGE_MASK) == 0, + ("pmap_invalidate_cache_range: eva not page-aligned")); + + if (cpu_feature & CPUID_SS) + ; /* If "Self Snoop" is supported, do nothing. */ + else if (cpu_feature & CPUID_CLFSH) { + + /* + * Otherwise, do per-cache line flush. Use the mfence + * instruction to insure that previous stores are + * included in the write-back. The processor + * propagates flush to other processors in the cache + * coherence domain. + */ + mfence(); + for (; sva < eva; sva += cpu_clflush_line_size) + clflush(sva); + mfence(); + } else { + + /* + * No targeted cache flush methods are supported by CPU, + * globally invalidate cache as a last resort. + */ + pmap_invalidate_cache(); + } +} + /* * Are we current address space or kernel? N.B. We return FALSE when * a pmap's page table is in use because a kernel thread is borrowing @@ -4422,7 +4457,8 @@ pmap_pde_attr(pd_entry_t *pde, int cache void * pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) { - vm_offset_t va, tmpva, offset; + vm_offset_t va, offset; + vm_size_t tmpsize; offset = pa & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); @@ -4435,16 +4471,10 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_ if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); - for (tmpva = va; size > 0; ) { - pmap_kenter_attr(tmpva, pa, mode); - size -= PAGE_SIZE; - tmpva += PAGE_SIZE; - pa += PAGE_SIZE; - } - pmap_invalidate_range(kernel_pmap, va, tmpva); - /* If "Self Snoop" is supported, do nothing. */ - if (!(cpu_feature & CPUID_SS)) - pmap_invalidate_cache(); + for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) + pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); + pmap_invalidate_range(kernel_pmap, va, va + tmpsize); + pmap_invalidate_cache_range(va, va + size); return ((void *)(va + offset)); } @@ -4484,16 +4514,48 @@ pmap_unmapdev(vm_offset_t va, vm_size_t void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { + struct sysmaps *sysmaps; + vm_offset_t sva, eva; m->md.pat_mode = ma; + if ((m->flags & PG_FICTITIOUS) != 0) + return; /* * If "m" is a normal page, flush it from the cache. + * See pmap_invalidate_cache_range(). + * + * First, try to find an existing mapping of the page by sf + * buffer. sf_buf_invalidate_cache() modifies mapping and + * flushes the cache. */ - if ((m->flags & PG_FICTITIOUS) == 0) { - /* If "Self Snoop" is supported, do nothing. */ - if (!(cpu_feature & CPUID_SS)) - pmap_invalidate_cache(); + if (sf_buf_invalidate_cache(m)) + return; + + /* + * If page is not mapped by sf buffer, but CPU does not + * support self snoop, map the page transient and do + * invalidation. In the worst case, whole cache is flushed by + * pmap_invalidate_cache_range(). + */ + if ((cpu_feature & (CPUID_SS|CPUID_CLFSH)) == CPUID_CLFSH) { + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + mtx_lock(&sysmaps->lock); + if (*sysmaps->CMAP2) + panic("pmap_page_set_memattr: CMAP2 busy"); + sched_pin(); + *sysmaps->CMAP2 = PG_V | PG_RW | VM_PAGE_TO_PHYS(m) | + PG_A | PG_M | pmap_cache_bits(m->md.pat_mode, 0); + invlcaddr(sysmaps->CADDR2); + sva = (vm_offset_t)sysmaps->CADDR2; + eva = sva + PAGE_SIZE; + } else + sva = eva = 0; /* gcc */ + pmap_invalidate_cache_range(sva, eva); + if (sva != 0) { + *sysmaps->CMAP2 = 0; + sched_unpin(); + mtx_unlock(&sysmaps->lock); } } @@ -4606,7 +4668,7 @@ pmap_change_attr(vm_offset_t va, vm_size */ if (changed) { pmap_invalidate_range(kernel_pmap, base, tmpva); - pmap_invalidate_cache(); + pmap_invalidate_cache_range(base, tmpva); } return (0); } Modified: stable/7/sys/i386/i386/vm_machdep.c ============================================================================== --- stable/7/sys/i386/i386/vm_machdep.c Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/i386/vm_machdep.c Tue Jan 26 20:58:09 2010 (r203047) @@ -710,6 +710,39 @@ sf_buf_init(void *arg) } /* + * Invalidate the cache lines that may belong to the page, if + * (possibly old) mapping of the page by sf buffer exists. Returns + * TRUE when mapping was found and cache invalidated. + */ +boolean_t +sf_buf_invalidate_cache(vm_page_t m) +{ + struct sf_head *hash_list; + struct sf_buf *sf; + boolean_t ret; + + hash_list = &sf_buf_active[SF_BUF_HASH(m)]; + ret = FALSE; + mtx_lock(&sf_buf_lock); + LIST_FOREACH(sf, hash_list, list_entry) { + if (sf->m == m) { + /* + * Use pmap_qenter to update the pte for + * existing mapping, in particular, the PAT + * settings are recalculated. + */ + pmap_qenter(sf->kva, &m, 1); + pmap_invalidate_cache_range(sf->kva, sf->kva + + PAGE_SIZE); + ret = TRUE; + break; + } + } + mtx_unlock(&sf_buf_lock); + return (ret); +} + +/* * Get an sf_buf from the freelist. May block if none are available. */ struct sf_buf * @@ -775,7 +808,8 @@ sf_buf_alloc(struct vm_page *m, int flag */ ptep = vtopte(sf->kva); opte = *ptep; - *ptep = VM_PAGE_TO_PHYS(m) | pgeflag | PG_RW | PG_V; + *ptep = VM_PAGE_TO_PHYS(m) | pgeflag | PG_RW | PG_V | + pmap_cache_bits(m->md.pat_mode, 0); /* * Avoid unnecessary TLB invalidations: If the sf_buf's old Modified: stable/7/sys/i386/include/cpufunc.h ============================================================================== --- stable/7/sys/i386/include/cpufunc.h Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/include/cpufunc.h Tue Jan 26 20:58:09 2010 (r203047) @@ -79,6 +79,13 @@ bsrl(u_int mask) } static __inline void +clflush(u_long addr) +{ + + __asm __volatile("clflush %0" : : "m" (*(char *)addr)); +} + +static __inline void disable_intr(void) { __asm __volatile("cli" : : : "memory"); @@ -106,6 +113,13 @@ enable_intr(void) __asm __volatile("sti"); } +static __inline void +mfence(void) +{ + + __asm __volatile("mfence" : : : "memory"); +} + #ifdef _KERNEL #define HAVE_INLINE_FFS Modified: stable/7/sys/i386/include/md_var.h ============================================================================== --- stable/7/sys/i386/include/md_var.h Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/include/md_var.h Tue Jan 26 20:58:09 2010 (r203047) @@ -52,6 +52,7 @@ extern u_int amd_feature2; extern u_int amd_pminfo; extern u_int via_feature_rng; extern u_int via_feature_xcrypt; +extern u_int cpu_clflush_line_size; extern u_int cpu_fxsr; extern u_int cpu_high; extern u_int cpu_id; Modified: stable/7/sys/i386/include/pmap.h ============================================================================== --- stable/7/sys/i386/include/pmap.h Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/include/pmap.h Tue Jan 26 20:58:09 2010 (r203047) @@ -415,6 +415,7 @@ extern vm_offset_t virtual_end; #define pmap_unmapbios(va, sz) pmap_unmapdev((va), (sz)) void pmap_bootstrap(vm_paddr_t); +int pmap_cache_bits(int mode, boolean_t is_pde); int pmap_change_attr(vm_offset_t, vm_size_t, int); void pmap_init_pat(void); void pmap_kenter(vm_offset_t va, vm_paddr_t pa); @@ -432,6 +433,7 @@ void pmap_invalidate_page(pmap_t, vm_off void pmap_invalidate_range(pmap_t, vm_offset_t, vm_offset_t); void pmap_invalidate_all(pmap_t); void pmap_invalidate_cache(void); +void pmap_invalidate_cache_range(vm_offset_t, vm_offset_t); #endif /* _KERNEL */ Modified: stable/7/sys/i386/include/sf_buf.h ============================================================================== --- stable/7/sys/i386/include/sf_buf.h Tue Jan 26 20:50:41 2010 (r203046) +++ stable/7/sys/i386/include/sf_buf.h Tue Jan 26 20:58:09 2010 (r203047) @@ -58,4 +58,6 @@ sf_buf_page(struct sf_buf *sf) return (sf->m); } +boolean_t sf_buf_invalidate_cache(vm_page_t m); + #endif /* !_MACHINE_SF_BUF_H_ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201001262058.o0QKw9T6002049>