Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 8 Nov 2018 22:39:38 +0000 (UTC)
From:      John Baldwin <jhb@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r340269 - in stable/12/sys: amd64/include arm/include arm64/include i386/include kern mips/include powerpc/include riscv/include sparc64/include x86/x86
Message-ID:  <201811082239.wA8MdcIM013489@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: jhb
Date: Thu Nov  8 22:39:38 2018
New Revision: 340269
URL: https://svnweb.freebsd.org/changeset/base/340269

Log:
  MFC 340164,340168,340170: Add custom cpu_lock_delay() for x86.
  
  340164:
  Add a KPI for the delay while spinning on a spin lock.
  
  Replace a call to DELAY(1) with a new cpu_lock_delay() KPI.  Currently
  cpu_lock_delay() is defined to DELAY(1) on all platforms.  However,
  platforms with a DELAY() implementation that uses spin locks should
  implement a custom cpu_lock_delay() doesn't use locks.
  
  340168:
  Add a delay_tsc() static function for when DELAY() uses the TSC.
  
  This uses slightly simpler logic than the existing code by using the
  full 64-bit counter and thus not having to worry about counter
  overflow.
  
  340170:
  Add a custom implementation of cpu_lock_delay() for x86.
  
  Avoid using DELAY() since it can try to use spin locks on CPUs without
  a P-state invariant TSC.  For cpu_lock_delay(), always use the TSC if
  it exists (even if it is not P-state invariant) to delay for a
  microsecond.  If the TSC does not exist, read from I/O port 0x84 to
  delay instead.
  
  PR:		228768
  Approved by:	re (gjb)

Modified:
  stable/12/sys/amd64/include/cpu.h
  stable/12/sys/arm/include/cpu.h
  stable/12/sys/arm64/include/cpu.h
  stable/12/sys/i386/include/cpu.h
  stable/12/sys/kern/kern_mutex.c
  stable/12/sys/mips/include/cpu.h
  stable/12/sys/powerpc/include/cpu.h
  stable/12/sys/riscv/include/cpu.h
  stable/12/sys/sparc64/include/cpu.h
  stable/12/sys/x86/x86/delay.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/amd64/include/cpu.h
==============================================================================
--- stable/12/sys/amd64/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/amd64/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -77,6 +77,7 @@ extern char	etext[];
 extern	void (*vmm_resume_p)(void);
 
 void	cpu_halt(void);
+void	cpu_lock_delay(void);
 void	cpu_reset(void);
 void	fork_trampoline(void);
 void	swi_vm(void *);

Modified: stable/12/sys/arm/include/cpu.h
==============================================================================
--- stable/12/sys/arm/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/arm/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -61,6 +61,7 @@ get_cyclecount(void)
 #define cpu_getstack(td)	((td)->td_frame->tf_usr_sp)
 #define cpu_setstack(td, sp)	((td)->td_frame->tf_usr_sp = (sp))
 #define cpu_spinwait()		/* nothing */
+#define	cpu_lock_delay()	DELAY(1)
 
 #define ARM_NVEC		8
 #define ARM_VEC_ALL		0xffffffff

Modified: stable/12/sys/arm64/include/cpu.h
==============================================================================
--- stable/12/sys/arm64/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/arm64/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -51,6 +51,7 @@
 #define	cpu_getstack(td)	((td)->td_frame->tf_sp)
 #define	cpu_setstack(td, sp)	((td)->td_frame->tf_sp = (sp))
 #define	cpu_spinwait()		__asm __volatile("yield" ::: "memory")
+#define	cpu_lock_delay()	DELAY(1)
 
 /* Extract CPU affinity levels 0-3 */
 #define	CPU_AFF0(mpidr)	(u_int)(((mpidr) >> 0) & 0xff)

Modified: stable/12/sys/i386/include/cpu.h
==============================================================================
--- stable/12/sys/i386/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/i386/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -72,6 +72,7 @@ extern char	btext[];
 extern char	etext[];
 
 void	cpu_halt(void);
+void	cpu_lock_delay(void);
 void	cpu_reset(void);
 void	fork_trampoline(void);
 void	swi_vm(void *);

Modified: stable/12/sys/kern/kern_mutex.c
==============================================================================
--- stable/12/sys/kern/kern_mutex.c	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/kern/kern_mutex.c	Thu Nov  8 22:39:38 2018	(r340269)
@@ -1206,7 +1206,7 @@ _mtx_lock_indefinite_check(struct mtx *m, struct lock_
 
 	ldap->spin_cnt++;
 	if (ldap->spin_cnt < 60000000 || kdb_active || panicstr != NULL)
-		DELAY(1);
+		cpu_lock_delay();
 	else {
 		td = mtx_owner(m);
 

Modified: stable/12/sys/mips/include/cpu.h
==============================================================================
--- stable/12/sys/mips/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/mips/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -71,6 +71,7 @@
 #define	cpu_getstack(td)	((td)->td_frame->sp)
 #define	cpu_setstack(td, nsp)	((td)->td_frame->sp = (nsp))
 #define	cpu_spinwait()		/* nothing */
+#define	cpu_lock_delay()	DELAY(1)
 
 /*
  * A machine-independent interface to the CPU's counter.

Modified: stable/12/sys/powerpc/include/cpu.h
==============================================================================
--- stable/12/sys/powerpc/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/powerpc/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -128,6 +128,7 @@ get_cyclecount(void)
 
 #define	cpu_getstack(td)	((td)->td_frame->fixreg[1])
 #define	cpu_spinwait()		__asm __volatile("or 27,27,27") /* yield */
+#define	cpu_lock_delay()	DELAY(1)
 
 extern char btext[];
 extern char etext[];

Modified: stable/12/sys/riscv/include/cpu.h
==============================================================================
--- stable/12/sys/riscv/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/riscv/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -46,6 +46,7 @@
 #define	cpu_getstack(td)	((td)->td_frame->tf_sp)
 #define	cpu_setstack(td, sp)	((td)->td_frame->tf_sp = (sp))
 #define	cpu_spinwait()		/* nothing */
+#define	cpu_lock_delay()	DELAY(1)
 
 #ifdef _KERNEL
 

Modified: stable/12/sys/sparc64/include/cpu.h
==============================================================================
--- stable/12/sys/sparc64/include/cpu.h	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/sparc64/include/cpu.h	Thu Nov  8 22:39:38 2018	(r340269)
@@ -48,6 +48,7 @@
 #define	cpu_getstack(td)	((td)->td_frame->tf_sp)
 #define	cpu_setstack(td, sp)	((td)->td_frame->tf_sp = (sp))
 #define	cpu_spinwait()		/* nothing */
+#define	cpu_lock_delay()	DELAY(1)
 
 #ifdef _KERNEL
 

Modified: stable/12/sys/x86/x86/delay.c
==============================================================================
--- stable/12/sys/x86/x86/delay.c	Thu Nov  8 21:58:51 2018	(r340268)
+++ stable/12/sys/x86/x86/delay.c	Thu Nov  8 22:39:38 2018	(r340269)
@@ -51,11 +51,23 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <x86/init.h>
 
-static u_int
-get_tsc(__unused struct timecounter *tc)
+static void
+delay_tsc(int n)
 {
+	uint64_t end, now;
 
-	return (rdtsc32());
+	/*
+	 * Pin the current thread ensure correct behavior if the TSCs
+	 * on different CPUs are not in sync.
+	 */
+	sched_pin();
+	now = rdtsc();
+	end = now + tsc_freq * n / 1000000;
+	do {
+		cpu_spinwait();
+		now = rdtsc();
+	} while (now < end);
+	sched_unpin();
 }
 
 static int
@@ -66,22 +78,24 @@ delay_tc(int n)
 	uint64_t end, freq, now;
 	u_int last, mask, u;
 
-	tc = timecounter;
-	freq = atomic_load_acq_64(&tsc_freq);
-	if (tsc_is_invariant && freq != 0) {
-		func = get_tsc;
-		mask = ~0u;
-	} else {
-		if (tc->tc_quality <= 0)
-			return (0);
-		func = tc->tc_get_timecount;
-		mask = tc->tc_counter_mask;
-		freq = tc->tc_frequency;
+	/*
+	 * Only use the TSC if it is P-state invariant.  If the TSC is
+	 * not P-state invariant and the CPU is not running at the
+	 * "full" P-state, then the TSC will increment at some rate
+	 * less than tsc_freq and delay_tsc() will wait too long.
+	 */
+	if (tsc_is_invariant && tsc_freq != 0) {
+		delay_tsc(n);
+		return (1);
 	}
+	tc = timecounter;
+	if (tc->tc_quality <= 0)
+		return (0);
+	func = tc->tc_get_timecount;
+	mask = tc->tc_counter_mask;
+	freq = tc->tc_frequency;
 	now = 0;
 	end = freq * n / 1000000;
-	if (func == get_tsc)
-		sched_pin();
 	last = func(tc) & mask;
 	do {
 		cpu_spinwait();
@@ -92,8 +106,6 @@ delay_tc(int n)
 			now += u - last;
 		last = u;
 	} while (now < end);
-	if (func == get_tsc)
-		sched_unpin();
 	return (1);
 }
 
@@ -109,4 +121,23 @@ DELAY(int n)
 
 	init_ops.early_delay(n);
 	TSEXIT();
+}
+
+void
+cpu_lock_delay(void)
+{
+
+	/*
+	 * Use TSC to wait for a usec if present, otherwise fall back
+	 * to reading from port 0x84.  We can't call into timecounters
+	 * for this delay since timecounters might use spin locks.
+	 *
+	 * Note that unlike delay_tc(), this uses the TSC even if it
+	 * is not P-state invariant.  For this function it is ok to
+	 * wait even a few usecs.
+	 */
+	if (tsc_freq != 0)
+		delay_tsc(1);
+	else
+		inb(0x84);
 }



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201811082239.wA8MdcIM013489>