Date: Fri, 20 Jan 2017 19:08:45 +0000 (UTC) From: Konstantin Belousov <kib@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r312555 - in head/sys: amd64/amd64 amd64/include i386/i386 i386/include Message-ID: <201701201908.v0KJ8jms084704@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kib Date: Fri Jan 20 19:08:44 2017 New Revision: 312555 URL: https://svnweb.freebsd.org/changeset/base/312555 Log: Use SFENCE for ordering CLFLUSHOPT. SDM states that CLFLUSHOPT instructions can be ordered with other writes by SFENCE, heavier MFENCE is not required. Reviewed by: alc Sponsored by: The FreeBSD Foundation MFC after: 2 weeks Modified: head/sys/amd64/amd64/pmap.c head/sys/amd64/include/cpufunc.h head/sys/i386/i386/pmap.c head/sys/i386/include/cpufunc.h Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Fri Jan 20 18:37:14 2017 (r312554) +++ head/sys/amd64/amd64/pmap.c Fri Jan 20 19:08:44 2017 (r312555) @@ -1862,16 +1862,16 @@ pmap_invalidate_cache_range(vm_offset_t return; /* - * Otherwise, do per-cache line flush. Use the mfence + * Otherwise, do per-cache line flush. Use the sfence * instruction to insure that previous stores are * included in the write-back. The processor * propagates flush to other processors in the cache * coherence domain. */ - mfence(); + sfence(); for (; sva < eva; sva += cpu_clflush_line_size) clflushopt(sva); - mfence(); + sfence(); } else if ((cpu_feature & CPUID_CLFSH) != 0 && eva - sva < PMAP_CLFLUSH_THRESHOLD) { if (pmap_kextract(sva) == lapic_paddr) @@ -1915,7 +1915,9 @@ pmap_invalidate_cache_pages(vm_page_t *p ((cpu_feature & CPUID_CLFSH) == 0 && !useclflushopt)) pmap_invalidate_cache(); else { - if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) + if (useclflushopt) + sfence(); + else if (cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); for (i = 0; i < count; i++) { daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i])); @@ -1927,7 +1929,9 @@ pmap_invalidate_cache_pages(vm_page_t *p clflush(daddr); } } - if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) + if (useclflushopt) + sfence(); + else if (cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); } } Modified: head/sys/amd64/include/cpufunc.h ============================================================================== --- head/sys/amd64/include/cpufunc.h Fri Jan 20 18:37:14 2017 (r312554) +++ head/sys/amd64/include/cpufunc.h Fri Jan 20 19:08:44 2017 (r312555) @@ -327,6 +327,13 @@ mfence(void) } static __inline void +sfence(void) +{ + + __asm __volatile("sfence" : : : "memory"); +} + +static __inline void ia32_pause(void) { __asm __volatile("pause"); Modified: head/sys/i386/i386/pmap.c ============================================================================== --- head/sys/i386/i386/pmap.c Fri Jan 20 18:37:14 2017 (r312554) +++ head/sys/i386/i386/pmap.c Fri Jan 20 19:08:44 2017 (r312555) @@ -1283,16 +1283,16 @@ pmap_invalidate_cache_range(vm_offset_t return; #endif /* - * Otherwise, do per-cache line flush. Use the mfence + * Otherwise, do per-cache line flush. Use the sfence * instruction to insure that previous stores are * included in the write-back. The processor * propagates flush to other processors in the cache * coherence domain. */ - mfence(); + sfence(); for (; sva < eva; sva += cpu_clflush_line_size) clflushopt(sva); - mfence(); + sfence(); } else if ((cpu_feature & CPUID_CLFSH) != 0 && eva - sva < PMAP_CLFLUSH_THRESHOLD) { #ifdef DEV_APIC @@ -5300,12 +5300,14 @@ pmap_flush_page(vm_page_t m) eva = sva + PAGE_SIZE; /* - * Use mfence despite the ordering implied by + * Use mfence or sfence despite the ordering implied by * mtx_{un,}lock() because clflush on non-Intel CPUs * and clflushopt are not guaranteed to be ordered by * any other instruction. */ - if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) + if (useclflushopt) + sfence(); + else if (cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); for (; sva < eva; sva += cpu_clflush_line_size) { if (useclflushopt) @@ -5313,7 +5315,9 @@ pmap_flush_page(vm_page_t m) else clflush(sva); } - if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) + if (useclflushopt) + sfence(); + else if (cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); *cmap_pte2 = 0; sched_unpin(); Modified: head/sys/i386/include/cpufunc.h ============================================================================== --- head/sys/i386/include/cpufunc.h Fri Jan 20 18:37:14 2017 (r312554) +++ head/sys/i386/include/cpufunc.h Fri Jan 20 19:08:44 2017 (r312555) @@ -158,6 +158,13 @@ mfence(void) __asm __volatile("mfence" : : : "memory"); } +static __inline void +sfence(void) +{ + + __asm __volatile("sfence" : : : "memory"); +} + #ifdef _KERNEL #define HAVE_INLINE_FFS
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201701201908.v0KJ8jms084704>