Skip site navigation (1)Skip section navigation (2) 
Date:      Sat, 7 Jul 2018 01:54:45 +0000 (UTC)
From:      Jeff Roberson <jeff@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r336055 - head/sys/vm
Message-ID:  <201807070154.w671sjZK070521@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: jeff
Date: Sat Jul  7 01:54:45 2018
New Revision: 336055
URL: https://svnweb.freebsd.org/changeset/base/336055

Log:
  Use the ticks since the last update to reduce hysteresis in the partpopq and
  contention on the vm_reserv_domain lock.
  
  This gives a roughly 8x speedup on will-it-scale fault1 on a 16 core machine.
  
  Reviewed by:	alc, kib, markj

Modified:
  head/sys/vm/vm_reserv.c

Modified: head/sys/vm/vm_reserv.c
==============================================================================
--- head/sys/vm/vm_reserv.c	Sat Jul  7 00:41:04 2018	(r336054)
+++ head/sys/vm/vm_reserv.c	Sat Jul  7 01:54:45 2018	(r336055)
@@ -116,6 +116,12 @@ typedef	u_long		popmap_t;
 #define	NPOPMAP		howmany(VM_LEVEL_0_NPAGES, NBPOPMAP)
 
 /*
+ * Number of elapsed ticks before we update the LRU queue position.  Used
+ * to reduce contention and churn on the list.
+ */
+#define	PARTPOPSLOP	1
+
+/*
  * Clear a bit in the population map.
  */
 static __inline void
@@ -183,6 +189,7 @@ struct vm_reserv {
 	vm_page_t	pages;			/* (c) first page  */
 	uint16_t	domain;			/* (c) NUMA domain. */
 	uint16_t	popcnt;			/* (r) # of pages in use */
+	int		lasttick;		/* (r) last pop update tick. */
 	char		inpartpopq;		/* (d) */
 	popmap_t	popmap[NPOPMAP];	/* (r) bit vector, used pages */
 };
@@ -394,6 +401,7 @@ vm_reserv_insert(vm_reserv_t rv, vm_object_t object, v
 	vm_reserv_object_lock(object);
 	rv->pindex = pindex;
 	rv->object = object;
+	rv->lasttick = ticks;
 	LIST_INSERT_HEAD(&object->rvq, rv, objq);
 	vm_reserv_object_unlock(object);
 }
@@ -430,16 +438,20 @@ vm_reserv_depopulate(vm_reserv_t rv, int index)
 	}
 	popmap_clear(rv->popmap, index);
 	rv->popcnt--;
-	vm_reserv_domain_lock(rv->domain);
-	if (rv->inpartpopq) {
-		TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
-		rv->inpartpopq = FALSE;
+	if ((unsigned)(ticks - rv->lasttick) >= PARTPOPSLOP ||
+	    rv->popcnt == 0) {
+		vm_reserv_domain_lock(rv->domain);
+		if (rv->inpartpopq) {
+			TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);
+			rv->inpartpopq = FALSE;
+		}
+		if (rv->popcnt != 0) {
+			rv->inpartpopq = TRUE;
+			TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq);
+		}
+		vm_reserv_domain_unlock(rv->domain);
+		rv->lasttick = ticks;
 	}
-	if (rv->popcnt != 0) {
-		rv->inpartpopq = TRUE;
-		TAILQ_INSERT_TAIL(&vm_rvq_partpop[rv->domain], rv, partpopq);
-	}
-	vm_reserv_domain_unlock(rv->domain);
 	vmd = VM_DOMAIN(rv->domain);
 	if (rv->popcnt == 0) {
 		vm_reserv_remove(rv);
@@ -536,6 +548,10 @@ vm_reserv_populate(vm_reserv_t rv, int index)
 	    rv, rv->domain));
 	popmap_set(rv->popmap, index);
 	rv->popcnt++;
+	if ((unsigned)(ticks - rv->lasttick) < PARTPOPSLOP &&
+	    rv->inpartpopq && rv->popcnt != VM_LEVEL_0_NPAGES)
+		return;
+	rv->lasttick = ticks;
 	vm_reserv_domain_lock(rv->domain);
 	if (rv->inpartpopq) {
 		TAILQ_REMOVE(&vm_rvq_partpop[rv->domain], rv, partpopq);

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201807070154.w671sjZK070521>