Skip site navigation (1)Skip section navigation (2) 
Date:      Mon, 8 Jul 2019 18:56:30 +0000 (UTC)
From:      Mark Johnston <markj@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r349840 - head/sys/vm
Message-ID:  <201907081856.x68IuUP2058723@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: markj
Date: Mon Jul  8 18:56:30 2019
New Revision: 349840
URL: https://svnweb.freebsd.org/changeset/base/349840

Log:
  Add a per-CPU page cache per VM free pool.
  
  Some workloads benefit from having a per-CPU cache for
  VM_FREEPOOL_DIRECT pages.
  
  Reviewed by:	dougm, kib
  Discussed with:	alc, jeff
  MFC after:	2 weeks
  Sponsored by:	Netflix
  Differential Revision:	https://reviews.freebsd.org/D20858

Modified:
  head/sys/vm/vm_page.c
  head/sys/vm/vm_page.h
  head/sys/vm/vm_pagequeue.h

Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c	Mon Jul  8 18:29:37 2019	(r349839)
+++ head/sys/vm/vm_page.c	Mon Jul  8 18:56:30 2019	(r349840)
@@ -192,21 +192,28 @@ static void
 vm_page_init_cache_zones(void *dummy __unused)
 {
 	struct vm_domain *vmd;
-	int i;
+	struct vm_pgcache *pgcache;
+	int domain, pool;
 
-	for (i = 0; i < vm_ndomains; i++) {
-		vmd = VM_DOMAIN(i);
+	for (domain = 0; domain < vm_ndomains; domain++) {
+		vmd = VM_DOMAIN(domain);
+
 		/*
-		 * Don't allow the page cache to take up more than .25% of
+		 * Don't allow the page caches to take up more than .25% of
 		 * memory.
 		 */
-		if (vmd->vmd_page_count / 400 < 256 * mp_ncpus)
+		if (vmd->vmd_page_count / 400 < 256 * mp_ncpus * VM_NFREEPOOL)
 			continue;
-		vmd->vmd_pgcache = uma_zcache_create("vm pgcache",
-		    sizeof(struct vm_page), NULL, NULL, NULL, NULL,
-		    vm_page_import, vm_page_release, vmd,
-		    UMA_ZONE_MAXBUCKET | UMA_ZONE_VM);
-		(void )uma_zone_set_maxcache(vmd->vmd_pgcache, 0);
+		for (pool = 0; pool < VM_NFREEPOOL; pool++) {
+			pgcache = &vmd->vmd_pgcache[pool];
+			pgcache->domain = domain;
+			pgcache->pool = pool;
+			pgcache->zone = uma_zcache_create("vm pgcache",
+			    sizeof(struct vm_page), NULL, NULL, NULL, NULL,
+			    vm_page_import, vm_page_release, pgcache,
+			    UMA_ZONE_MAXBUCKET | UMA_ZONE_VM);
+			(void)uma_zone_set_maxcache(pgcache->zone, 0);
+		}
 	}
 }
 SYSINIT(vm_page2, SI_SUB_VM_CONF, SI_ORDER_ANY, vm_page_init_cache_zones, NULL);
@@ -1797,7 +1804,7 @@ vm_page_alloc_domain_after(vm_object_t object, vm_pind
 {
 	struct vm_domain *vmd;
 	vm_page_t m;
-	int flags;
+	int flags, pool;
 
 	KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0) &&
 	    (object != NULL || (req & VM_ALLOC_SBUSY) == 0) &&
@@ -1814,6 +1821,7 @@ vm_page_alloc_domain_after(vm_object_t object, vm_pind
 
 	flags = 0;
 	m = NULL;
+	pool = object != NULL ? VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT;
 again:
 #if VM_NRESERVLEVEL > 0
 	/*
@@ -1828,8 +1836,8 @@ again:
 	}
 #endif
 	vmd = VM_DOMAIN(domain);
-	if (object != NULL && vmd->vmd_pgcache != NULL) {
-		m = uma_zalloc(vmd->vmd_pgcache, M_NOWAIT);
+	if (vmd->vmd_pgcache[pool].zone != NULL) {
+		m = uma_zalloc(vmd->vmd_pgcache[pool].zone, M_NOWAIT);
 		if (m != NULL) {
 			flags |= PG_PCPU_CACHE;
 			goto found;
@@ -1840,8 +1848,7 @@ again:
 		 * If not, allocate it from the free page queues.
 		 */
 		vm_domain_free_lock(vmd);
-		m = vm_phys_alloc_pages(domain, object != NULL ?
-		    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
+		m = vm_phys_alloc_pages(domain, pool, 0);
 		vm_domain_free_unlock(vmd);
 		if (m == NULL) {
 			vm_domain_freecnt_inc(vmd, 1);
@@ -2231,15 +2238,17 @@ static int
 vm_page_import(void *arg, void **store, int cnt, int domain, int flags)
 {
 	struct vm_domain *vmd;
+	struct vm_pgcache *pgcache;
 	int i;
 
-	vmd = arg;
+	pgcache = arg;
+	vmd = VM_DOMAIN(pgcache->domain);
 	/* Only import if we can bring in a full bucket. */
 	if (cnt == 1 || !vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt))
 		return (0);
 	domain = vmd->vmd_domain;
 	vm_domain_free_lock(vmd);
-	i = vm_phys_alloc_npages(domain, VM_FREEPOOL_DEFAULT, cnt,
+	i = vm_phys_alloc_npages(domain, pgcache->pool, cnt,
 	    (vm_page_t *)store);
 	vm_domain_free_unlock(vmd);
 	if (cnt != i)
@@ -2252,10 +2261,12 @@ static void
 vm_page_release(void *arg, void **store, int cnt)
 {
 	struct vm_domain *vmd;
+	struct vm_pgcache *pgcache;
 	vm_page_t m;
 	int i;
 
-	vmd = arg;
+	pgcache = arg;
+	vmd = VM_DOMAIN(pgcache->domain);
 	vm_domain_free_lock(vmd);
 	for (i = 0; i < cnt; i++) {
 		m = (vm_page_t)store[i];
@@ -3500,13 +3511,15 @@ void
 vm_page_free_toq(vm_page_t m)
 {
 	struct vm_domain *vmd;
+	uma_zone_t zone;
 
 	if (!vm_page_free_prep(m))
 		return;
 
 	vmd = vm_pagequeue_domain(m);
-	if ((m->flags & PG_PCPU_CACHE) != 0 && vmd->vmd_pgcache != NULL) {
-		uma_zfree(vmd->vmd_pgcache, m);
+	zone = vmd->vmd_pgcache[m->pool].zone;
+	if ((m->flags & PG_PCPU_CACHE) != 0 && zone != NULL) {
+		uma_zfree(zone, m);
 		return;
 	}
 	vm_domain_free_lock(vmd);

Modified: head/sys/vm/vm_page.h
==============================================================================
--- head/sys/vm/vm_page.h	Mon Jul  8 18:29:37 2019	(r349839)
+++ head/sys/vm/vm_page.h	Mon Jul  8 18:56:30 2019	(r349840)
@@ -378,6 +378,10 @@ extern struct mtx_padalign pa_lock[];
 /*
  * Page flags.  If changed at any other time than page allocation or
  * freeing, the modification must be protected by the vm_page lock.
+ *
+ * The PG_PCPU_CACHE flag is set at allocation time if the page was
+ * allocated from a per-CPU cache.  It is cleared the next time that the
+ * page is allocated from the physical memory allocator.
  */
 #define	PG_PCPU_CACHE	0x0001		/* was allocated from per-CPU caches */
 #define	PG_FICTITIOUS	0x0004		/* physical page doesn't exist */

Modified: head/sys/vm/vm_pagequeue.h
==============================================================================
--- head/sys/vm/vm_pagequeue.h	Mon Jul  8 18:29:37 2019	(r349839)
+++ head/sys/vm/vm_pagequeue.h	Mon Jul  8 18:56:30 2019	(r349840)
@@ -103,7 +103,11 @@ struct vm_domain {
 	struct vm_pagequeue vmd_pagequeues[PQ_COUNT];
 	struct mtx_padalign vmd_free_mtx;
 	struct mtx_padalign vmd_pageout_mtx;
-	uma_zone_t vmd_pgcache;		/* (c) page free cache. */
+	struct vm_pgcache {
+		int domain;
+		int pool;
+		uma_zone_t zone;
+	} vmd_pgcache[VM_NFREEPOOL];
 	struct vmem *vmd_kernel_arena;	/* (c) per-domain kva R/W arena. */
 	struct vmem *vmd_kernel_rwx_arena; /* (c) per-domain kva R/W/X arena. */
 	u_int vmd_domain;		/* (c) Domain number. */

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201907081856.x68IuUP2058723>