Skip site navigation (1)Skip section navigation (2) 
Date:      Wed, 22 Jul 2009 14:32:38 +0000 (UTC)
From:      Konstantin Belousov <kib@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r195820 - in head/sys/amd64: amd64 include
Message-ID:  <200907221432.n6MEWcXB043644@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: kib
Date: Wed Jul 22 14:32:38 2009
New Revision: 195820
URL: http://svn.freebsd.org/changeset/base/195820

Log:
  When the page caching attributes are changed, after new mapping is
  established, OS shall flush the caches on all processors that may have
  used the mapping previously. This operation is not needed if processors
  support self-snooping. If not, but clflush instruction is implemented
  on the CPU, series of the clflush can be used on the mapping region.
  Otherwise, we have to flush the whole cache. The later operation is very
  expensive, and AMD-made CPUs do not have self-snooping.
  
  Implement cache flush for remapped region by using clflush for amd64,
  when supported by CPU.
  
  Proposed and reviewed by:	alc
  Approved by:	re (kensmith)

Modified:
  head/sys/amd64/amd64/initcpu.c
  head/sys/amd64/amd64/pmap.c
  head/sys/amd64/include/cpufunc.h
  head/sys/amd64/include/md_var.h

Modified: head/sys/amd64/amd64/initcpu.c
==============================================================================
--- head/sys/amd64/amd64/initcpu.c	Wed Jul 22 14:32:28 2009	(r195819)
+++ head/sys/amd64/amd64/initcpu.c	Wed Jul 22 14:32:38 2009	(r195820)
@@ -65,6 +65,7 @@ char	cpu_vendor[20];		/* CPU Origin code
 u_int	cpu_vendor_id;		/* CPU vendor ID */
 u_int	cpu_fxsr;		/* SSE enabled */
 u_int	cpu_mxcsr_mask;		/* Valid bits in mxcsr */
+u_int	cpu_clflush_line_size = 32;
 
 SYSCTL_UINT(_hw, OID_AUTO, via_feature_rng, CTLFLAG_RD,
 	&via_feature_rng, 0, "VIA C3/C7 RNG feature available in CPU");
@@ -156,4 +157,12 @@ initializecpu(void)
 	    AMD64_CPU_FAMILY(cpu_id) == 0x6 &&
 	    AMD64_CPU_MODEL(cpu_id) >= 0xf)
 		init_via();
+
+	/*
+	 * CPUID with %eax = 1, %ebx returns
+	 * Bits 15-8: CLFLUSH line size
+	 * 	(Value * 8 = cache line size in bytes)
+	 */
+	if ((cpu_feature & CPUID_CLFSH) != 0)
+		cpu_clflush_line_size = ((cpu_procinfo >> 8) & 0xff) * 8;
 }

Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c	Wed Jul 22 14:32:28 2009	(r195819)
+++ head/sys/amd64/amd64/pmap.c	Wed Jul 22 14:32:38 2009	(r195820)
@@ -231,6 +231,7 @@ static vm_page_t pmap_enter_quick_locked
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
 static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte);
 static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte);
+static void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva);
 static boolean_t pmap_is_modified_pvh(struct md_page *pvh);
 static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
 static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
@@ -921,6 +922,40 @@ pmap_invalidate_cache(void)
 }
 #endif /* !SMP */
 
+static void
+pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva)
+{
+
+	KASSERT((sva & PAGE_MASK) == 0,
+	    ("pmap_invalidate_cache_range: sva not page-aligned"));
+	KASSERT((eva & PAGE_MASK) == 0,
+	    ("pmap_invalidate_cache_range: eva not page-aligned"));
+
+	if (cpu_feature & CPUID_SS)
+		; /* If "Self Snoop" is supported, do nothing. */
+	else if (cpu_feature & CPUID_CLFSH) {
+
+		/*
+		 * Otherwise, do per-cache line flush.  Use the mfence
+		 * instruction to insure that previous stores are
+		 * included in the write-back.  The processor
+		 * propagates flush to other processors in the cache
+		 * coherence domain.
+		 */
+		mfence();
+		for (; eva < sva; eva += cpu_clflush_line_size)
+			clflush(eva);
+		mfence();
+	} else {
+
+		/*
+		 * No targeted cache flush methods are supported by CPU,
+		 * globally invalidate cache as a last resort.
+		 */
+		pmap_invalidate_cache();
+	}
+}
+
 /*
  * Are we current address space or kernel?
  */
@@ -4256,7 +4291,8 @@ pmap_pde_attr(pd_entry_t *pde, int cache
 void *
 pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode)
 {
-	vm_offset_t va, tmpva, offset;
+	vm_offset_t va, offset;
+	vm_size_t tmpsize;
 
 	/*
 	 * If the specified range of physical addresses fits within the direct
@@ -4273,16 +4309,10 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_
 	if (!va)
 		panic("pmap_mapdev: Couldn't alloc kernel virtual memory");
 	pa = trunc_page(pa);
-	for (tmpva = va; size > 0; ) {
-		pmap_kenter_attr(tmpva, pa, mode);
-		size -= PAGE_SIZE;
-		tmpva += PAGE_SIZE;
-		pa += PAGE_SIZE;
-	}
-	pmap_invalidate_range(kernel_pmap, va, tmpva);
-	/* If "Self Snoop" is supported, do nothing. */
-	if (!(cpu_feature & CPUID_SS))
-		pmap_invalidate_cache();
+	for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE)
+		pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode);
+	pmap_invalidate_range(kernel_pmap, va, va + tmpsize);
+	pmap_invalidate_cache_range(va, va + tmpsize);
 	return ((void *)(va + offset));
 }
 
@@ -4624,9 +4654,7 @@ pmap_change_attr_locked(vm_offset_t va, 
 	 */
 	if (changed) {
 		pmap_invalidate_range(kernel_pmap, base, tmpva);
-		/* If "Self Snoop" is supported, do nothing. */
-		if (!(cpu_feature & CPUID_SS))
-			pmap_invalidate_cache();
+		pmap_invalidate_cache_range(base, tmpva);
 	}
 	return (error);
 }

Modified: head/sys/amd64/include/cpufunc.h
==============================================================================
--- head/sys/amd64/include/cpufunc.h	Wed Jul 22 14:32:28 2009	(r195819)
+++ head/sys/amd64/include/cpufunc.h	Wed Jul 22 14:32:38 2009	(r195820)
@@ -100,6 +100,13 @@ bsrq(u_long mask)
 }
 
 static __inline void
+clflush(u_long addr)
+{
+
+	__asm __volatile("clflush %0" : : "m" (*(char *)addr));
+}
+
+static __inline void
 disable_intr(void)
 {
 	__asm __volatile("cli" : : : "memory");
@@ -267,6 +274,13 @@ outw(u_int port, u_short data)
 }
 
 static __inline void
+mfence(void)
+{
+
+	__asm__ __volatile("mfence" : : : "memory");
+}
+
+static __inline void
 ia32_pause(void)
 {
 	__asm __volatile("pause");

Modified: head/sys/amd64/include/md_var.h
==============================================================================
--- head/sys/amd64/include/md_var.h	Wed Jul 22 14:32:28 2009	(r195819)
+++ head/sys/amd64/include/md_var.h	Wed Jul 22 14:32:38 2009	(r195820)
@@ -47,6 +47,7 @@ extern	u_int	amd_feature2;
 extern	u_int	amd_pminfo;
 extern	u_int	via_feature_rng;
 extern	u_int	via_feature_xcrypt;
+extern	u_int	cpu_clflush_line_size;
 extern	u_int	cpu_fxsr;
 extern	u_int	cpu_high;
 extern	u_int	cpu_id;

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200907221432.n6MEWcXB043644>