From owner-svn-src-all@freebsd.org Tue Aug 25 21:36:57 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 2565F3BCFC5; Tue, 25 Aug 2020 21:36:57 +0000 (UTC) (envelope-from cem@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4Bbj4T0D97z4CMb; Tue, 25 Aug 2020 21:36:57 +0000 (UTC) (envelope-from cem@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id DFFA78827; Tue, 25 Aug 2020 21:36:56 +0000 (UTC) (envelope-from cem@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 07PLauZ1061712; Tue, 25 Aug 2020 21:36:56 GMT (envelope-from cem@FreeBSD.org) Received: (from cem@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 07PLaudm061711; Tue, 25 Aug 2020 21:36:56 GMT (envelope-from cem@FreeBSD.org) Message-Id: <202008252136.07PLaudm061711@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: cem set sender to cem@FreeBSD.org using -f From: Conrad Meyer Date: Tue, 25 Aug 2020 21:36:56 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r364786 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: cem X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 364786 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.33 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 25 Aug 2020 21:36:57 -0000 Author: cem Date: Tue Aug 25 21:36:56 2020 New Revision: 364786 URL: https://svnweb.freebsd.org/changeset/base/364786 Log: vm_pageout: Scale worker threads with CPUs Autoscale vm_pageout worker threads from r364129 with CPU count. The default is arbitrarily chosen to be 16 CPUs per worker thread, but can be adjusted with the vm.pageout_cpus_per_thread tunable. There will never be less than 1 thread per populated NUMA domain, and the previous arbitrary upper limit (at most ncpus/2 threads per NUMA domain) is preserved. Care is taken to gracefully handle asymmetric NUMA nodes, such as empty node systems (e.g., AMD 2990WX) and systems with nodes of varying size (e.g., some larger >20 core Intel Haswell/Broadwell Xeon). Reviewed by: kib, markj Sponsored by: Isilon Differential Revision: https://reviews.freebsd.org/D26152 Modified: head/sys/vm/vm_pageout.c Modified: head/sys/vm/vm_pageout.c ============================================================================== --- head/sys/vm/vm_pageout.c Tue Aug 25 21:07:27 2020 (r364785) +++ head/sys/vm/vm_pageout.c Tue Aug 25 21:36:56 2020 (r364786) @@ -165,11 +165,10 @@ SYSCTL_INT(_vm, OID_AUTO, pageout_update_period, CTLFLAG_RWTUN, &vm_pageout_update_period, 0, "Maximum active LRU update period"); -/* Access with get_pageout_threads_per_domain(). */ -static int pageout_threads_per_domain = 1; -SYSCTL_INT(_vm, OID_AUTO, pageout_threads_per_domain, CTLFLAG_RDTUN, - &pageout_threads_per_domain, 0, - "Number of worker threads comprising each per-domain pagedaemon"); +static int pageout_cpus_per_thread = 16; +SYSCTL_INT(_vm, OID_AUTO, pageout_cpus_per_thread, CTLFLAG_RDTUN, + &pageout_cpus_per_thread, 0, + "Number of CPUs per pagedaemon worker thread"); SYSCTL_INT(_vm, OID_AUTO, lowmem_period, CTLFLAG_RWTUN, &lowmem_period, 0, "Low memory callback period"); @@ -2200,38 +2199,38 @@ vm_pageout_helper(void *arg) } static int -get_pageout_threads_per_domain(void) +get_pageout_threads_per_domain(const struct vm_domain *vmd) { - static bool resolved = false; - int half_cpus_per_dom; + unsigned total_pageout_threads, eligible_cpus, domain_cpus; - /* - * This is serialized externally by the sorted autoconfig portion of - * boot. - */ - if (__predict_true(resolved)) - return (pageout_threads_per_domain); + if (VM_DOMAIN_EMPTY(vmd->vmd_domain)) + return (0); /* * Semi-arbitrarily constrain pagedaemon threads to less than half the - * total number of threads in the system as an insane upper limit. + * total number of CPUs in the system as an upper limit. */ - half_cpus_per_dom = howmany(mp_ncpus / vm_ndomains, 2); + if (pageout_cpus_per_thread < 2) + pageout_cpus_per_thread = 2; + else if (pageout_cpus_per_thread > mp_ncpus) + pageout_cpus_per_thread = mp_ncpus; - if (pageout_threads_per_domain < 1) { - printf("Invalid tuneable vm.pageout_threads_per_domain value: " - "%d out of valid range: [1-%d]; clamping to 1\n", - pageout_threads_per_domain, half_cpus_per_dom); - pageout_threads_per_domain = 1; - } else if (pageout_threads_per_domain > half_cpus_per_dom) { - printf("Invalid tuneable vm.pageout_threads_per_domain value: " - "%d out of valid range: [1-%d]; clamping to %d\n", - pageout_threads_per_domain, half_cpus_per_dom, - half_cpus_per_dom); - pageout_threads_per_domain = half_cpus_per_dom; - } - resolved = true; - return (pageout_threads_per_domain); + total_pageout_threads = howmany(mp_ncpus, pageout_cpus_per_thread); + domain_cpus = CPU_COUNT(&cpuset_domain[vmd->vmd_domain]); + + /* Pagedaemons are not run in empty domains. */ + eligible_cpus = mp_ncpus; + for (unsigned i = 0; i < vm_ndomains; i++) + if (VM_DOMAIN_EMPTY(i)) + eligible_cpus -= CPU_COUNT(&cpuset_domain[i]); + + /* + * Assign a portion of the total pageout threads to this domain + * corresponding to the fraction of pagedaemon-eligible CPUs in the + * domain. In asymmetric NUMA systems, domains with more CPUs may be + * allocated more threads than domains with fewer CPUs. + */ + return (howmany(total_pageout_threads * domain_cpus, eligible_cpus)); } /* @@ -2288,7 +2287,7 @@ vm_pageout_init_domain(int domain) "pidctrl", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); pidctrl_init_sysctl(&vmd->vmd_pid, SYSCTL_CHILDREN(oid)); - vmd->vmd_inactive_threads = get_pageout_threads_per_domain(); + vmd->vmd_inactive_threads = get_pageout_threads_per_domain(vmd); } static void @@ -2343,7 +2342,6 @@ vm_pageout(void) p = curproc; td = curthread; - pageout_threads = get_pageout_threads_per_domain(); mtx_init(&vm_oom_ratelim_mtx, "vmoomr", NULL, MTX_DEF); swap_pager_swap_init(); @@ -2363,6 +2361,7 @@ vm_pageout(void) panic("starting pageout for domain %d: %d\n", i, error); } + pageout_threads = VM_DOMAIN(i)->vmd_inactive_threads; for (j = 0; j < pageout_threads - 1; j++) { error = kthread_add(vm_pageout_helper, (void *)(uintptr_t)i, p, NULL, 0, 0,