Skip site navigation (1)Skip section navigation (2) 
Date:      Tue, 28 Nov 2017 17:46:03 +0000 (UTC)
From:      Alan Cox <alc@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r326329 - head/sys/vm
Message-ID:  <201711281746.vASHk3SS096266@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: alc
Date: Tue Nov 28 17:46:03 2017
New Revision: 326329
URL: https://svnweb.freebsd.org/changeset/base/326329

Log:
  When the swap pager allocates space on disk, it requests contiguous
  blocks in a single call to blist_alloc().  However, when it frees
  that space, it previously called blist_free() on each block, one at a
  time.  With this change, the swap pager identifies ranges of
  contiguous blocks to be freed, and calls blist_free() once per
  range.  In one extreme case, that is described in the review, the time
  to perform an munmap(2) was reduced by 55%.
  
  Submitted by:	Doug Moore <dougm@rice.edu>
  Reviewed by:	kib
  MFC after:	1 week
  Differential Revision:	https://reviews.freebsd.org/D12397

Modified:
  head/sys/vm/swap_pager.c

Modified: head/sys/vm/swap_pager.c
==============================================================================
--- head/sys/vm/swap_pager.c	Tue Nov 28 17:33:10 2017	(r326328)
+++ head/sys/vm/swap_pager.c	Tue Nov 28 17:46:03 2017	(r326329)
@@ -390,6 +390,7 @@ SYSCTL_INT(_vm, OID_AUTO, dmmax, CTLFLAG_RD, &nsw_clus
 
 static void	swp_sizecheck(void);
 static void	swp_pager_async_iodone(struct buf *bp);
+static bool	swp_pager_swblk_empty(struct swblk *sb, int start, int limit);
 static int	swapongeom(struct vnode *);
 static int	swaponvp(struct thread *, struct vnode *, u_long);
 static int	swapoff_one(struct swdevt *sp, struct ucred *cred);
@@ -397,7 +398,7 @@ static int	swapoff_one(struct swdevt *sp, struct ucred
 /*
  * Swap bitmap functions
  */
-static void	swp_pager_freeswapspace(daddr_t blk, int npages);
+static void	swp_pager_freeswapspace(daddr_t blk, daddr_t npages);
 static daddr_t	swp_pager_getswapspace(int npages);
 
 /*
@@ -768,10 +769,12 @@ swp_pager_strategy(struct buf *bp)
  *	This routine may not sleep.
  */
 static void
-swp_pager_freeswapspace(daddr_t blk, int npages)
+swp_pager_freeswapspace(daddr_t blk, daddr_t npages)
 {
 	struct swdevt *sp;
 
+	if (npages == 0)
+		return;
 	mtx_lock(&sw_dev_mtx);
 	TAILQ_FOREACH(sp, &swtailq, sw_list) {
 		if (blk >= sp->sw_first && blk < sp->sw_end) {
@@ -1761,6 +1764,22 @@ next_obj:
  */
 
 /*
+ * SWP_PAGER_SWBLK_EMPTY() - is a range of blocks free?
+ */
+static bool
+swp_pager_swblk_empty(struct swblk *sb, int start, int limit)
+{
+	int i;
+
+	MPASS(0 <= start && start <= limit && limit <= SWAP_META_PAGES);
+	for (i = start; i < limit; i++) {
+		if (sb->d[i] != SWAPBLK_NONE)
+			return (false);
+	}
+	return (true);
+}
+   
+/*
  * SWP_PAGER_META_BUILD() -	add swap block to swap meta data for object
  *
  *	We first convert the object to a swap object if it is a default
@@ -1876,16 +1895,10 @@ allocated:
 	/*
 	 * Free the swblk if we end up with the empty page run.
 	 */
-	if (swapblk == SWAPBLK_NONE) {
-		for (i = 0; i < SWAP_META_PAGES; i++) {
-			if (sb->d[i] != SWAPBLK_NONE)
-				break;
-		}
-		if (i == SWAP_META_PAGES) {
-			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
-			    rdpi);
-			uma_zfree(swblk_zone, sb);
-		}
+	if (swapblk == SWAPBLK_NONE &&
+	    swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
+		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, rdpi);
+		uma_zfree(swblk_zone, sb);
 	}
 }
 
@@ -1903,37 +1916,46 @@ static void
 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count)
 {
 	struct swblk *sb;
+	daddr_t first_free, num_free;
 	vm_pindex_t last;
-	int i;
-	bool empty;
+	int i, limit, start;
 
 	VM_OBJECT_ASSERT_WLOCKED(object);
 	if (object->type != OBJT_SWAP || count == 0)
 		return;
 
-	last = pindex + count - 1;
+	first_free = SWAPBLK_NONE;
+	num_free = 0;
+	last = pindex + count;
 	for (;;) {
 		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,
 		    rounddown(pindex, SWAP_META_PAGES));
-		if (sb == NULL || sb->p > last)
+		if (sb == NULL || sb->p >= last)
 			break;
-		empty = true;
-		for (i = 0; i < SWAP_META_PAGES; i++) {
+		start = pindex > sb->p ? pindex - sb->p : 0;
+		limit = last - sb->p < SWAP_META_PAGES ? last - sb->p :
+		    SWAP_META_PAGES;
+		for (i = start; i < limit; i++) {
 			if (sb->d[i] == SWAPBLK_NONE)
 				continue;
-			if (pindex <= sb->p + i && sb->p + i <= last) {
-				swp_pager_freeswapspace(sb->d[i], 1);
-				sb->d[i] = SWAPBLK_NONE;
-			} else
-				empty = false;
+			if (first_free + num_free == sb->d[i])
+				num_free++;
+			else {
+				swp_pager_freeswapspace(first_free, num_free);
+				first_free = sb->d[i];
+				num_free = 1;
+			}
+			sb->d[i] = SWAPBLK_NONE;
 		}
-		pindex = sb->p + SWAP_META_PAGES;
-		if (empty) {
+		if (swp_pager_swblk_empty(sb, 0, start) &&
+		    swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) {
 			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
 			    sb->p);
 			uma_zfree(swblk_zone, sb);
 		}
+		pindex = sb->p + SWAP_META_PAGES;
 	}
+	swp_pager_freeswapspace(first_free, num_free);
 }
 
 /*
@@ -1946,6 +1968,7 @@ static void
 swp_pager_meta_free_all(vm_object_t object)
 {
 	struct swblk *sb;
+	daddr_t first_free, num_free;
 	vm_pindex_t pindex;
 	int i;
 
@@ -1953,16 +1976,26 @@ swp_pager_meta_free_all(vm_object_t object)
 	if (object->type != OBJT_SWAP)
 		return;
 
+	first_free = SWAPBLK_NONE;
+	num_free = 0;
 	for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
 	    &object->un_pager.swp.swp_blks, pindex)) != NULL;) {
 		pindex = sb->p + SWAP_META_PAGES;
 		for (i = 0; i < SWAP_META_PAGES; i++) {
-			if (sb->d[i] != SWAPBLK_NONE)
-				swp_pager_freeswapspace(sb->d[i], 1);
+			if (sb->d[i] == SWAPBLK_NONE)
+				continue;
+			if (first_free + num_free == sb->d[i])
+				num_free++;
+			else {
+				swp_pager_freeswapspace(first_free, num_free);
+				first_free = sb->d[i];
+				num_free = 1;
+			}
 		}
 		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);
 		uma_zfree(swblk_zone, sb);
 	}
+	swp_pager_freeswapspace(first_free, num_free);
 }
 
 /*
@@ -1987,7 +2020,6 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pin
 {
 	struct swblk *sb;
 	daddr_t r1;
-	int i;
 
 	if ((flags & (SWM_FREE | SWM_POP)) != 0)
 		VM_OBJECT_ASSERT_WLOCKED(object);
@@ -2010,11 +2042,7 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pin
 		return (SWAPBLK_NONE);
 	if ((flags & (SWM_FREE | SWM_POP)) != 0) {
 		sb->d[pindex % SWAP_META_PAGES] = SWAPBLK_NONE;
-		for (i = 0; i < SWAP_META_PAGES; i++) {
-			if (sb->d[i] != SWAPBLK_NONE)
-				break;
-		}
-		if (i == SWAP_META_PAGES) {
+		if (swp_pager_swblk_empty(sb, 0, SWAP_META_PAGES)) {
 			SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks,
 			    rounddown(pindex, SWAP_META_PAGES));
 			uma_zfree(swblk_zone, sb);

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201711281746.vASHk3SS096266>