Skip site navigation (1)Skip section navigation (2) 
Date:      Thu, 24 Oct 2019 19:12:01 +0000 (UTC)
From:      Alexander Motin <mav@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r354033 - stable/12/sys/kern
Message-ID:  <201910241912.x9OJC1GX006655@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: mav
Date: Thu Oct 24 19:12:01 2019
New Revision: 354033
URL: https://svnweb.freebsd.org/changeset/base/354033

Log:
  MFC r352728: Microoptimize sched_pickcpu() CPU affinity on SMT.
  
  Use of CPU_FFS() to implement CPUSET_FOREACH() allows to save up to ~0.5%
  of CPU time on 72-thread SMT system doing 80K IOPS to NVMe from one thread.

Modified:
  stable/12/sys/kern/sched_ule.c
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/kern/sched_ule.c
==============================================================================
--- stable/12/sys/kern/sched_ule.c	Thu Oct 24 19:11:01 2019	(r354032)
+++ stable/12/sys/kern/sched_ule.c	Thu Oct 24 19:12:01 2019	(r354033)
@@ -643,10 +643,6 @@ struct cpu_search {
 #define	CPU_SEARCH_HIGHEST	0x2
 #define	CPU_SEARCH_BOTH		(CPU_SEARCH_LOWEST|CPU_SEARCH_HIGHEST)
 
-#define	CPUSET_FOREACH(cpu, mask)				\
-	for ((cpu) = 0; (cpu) <= mp_maxid; (cpu)++)		\
-		if (CPU_ISSET(cpu, &mask))
-
 static __always_inline int cpu_search(const struct cpu_group *cg,
     struct cpu_search *low, struct cpu_search *high, const int match);
 int __noinline cpu_search_lowest(const struct cpu_group *cg,
@@ -1292,13 +1288,17 @@ sched_pickcpu(struct thread *td, int flags)
 	    tdq->tdq_lowpri >= PRI_MIN_IDLE &&
 	    SCHED_AFFINITY(ts, CG_SHARE_L2)) {
 		if (cg->cg_flags & CG_FLAG_THREAD) {
-			CPUSET_FOREACH(cpu, cg->cg_mask) {
-				if (TDQ_CPU(cpu)->tdq_lowpri < PRI_MIN_IDLE)
+			/* Check all SMT threads for being idle. */
+			for (cpu = CPU_FFS(&cg->cg_mask) - 1; ; cpu++) {
+				if (CPU_ISSET(cpu, &cg->cg_mask) &&
+				    TDQ_CPU(cpu)->tdq_lowpri < PRI_MIN_IDLE)
 					break;
+				if (cpu >= mp_maxid) {
+					SCHED_STAT_INC(pickcpu_idle_affinity);
+					return (ts->ts_cpu);
+				}
 			}
-		} else
-			cpu = INT_MAX;
-		if (cpu > mp_maxid) {
+		} else {
 			SCHED_STAT_INC(pickcpu_idle_affinity);
 			return (ts->ts_cpu);
 		}

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201910241912.x9OJC1GX006655>