From owner-svn-src-head@freebsd.org Mon Nov 9 23:05:28 2020 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 968C82D0010; Mon, 9 Nov 2020 23:05:28 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4CVRRX3skRz4VMG; Mon, 9 Nov 2020 23:05:28 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 7783619197; Mon, 9 Nov 2020 23:05:28 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 0A9N5Sit015930; Mon, 9 Nov 2020 23:05:28 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 0A9N5SYe015929; Mon, 9 Nov 2020 23:05:28 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <202011092305.0A9N5SYe015929@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Mon, 9 Nov 2020 23:05:28 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r367537 - head/sys/kern X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: head/sys/kern X-SVN-Commit-Revision: 367537 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.34 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 09 Nov 2020 23:05:28 -0000 Author: mjg Date: Mon Nov 9 23:05:28 2020 New Revision: 367537 URL: https://svnweb.freebsd.org/changeset/base/367537 Log: threads: reimplement tid allocation on top of a bitmap There are workloads with very bursty tid allocation and since unr tries very hard to have small-sized bitmaps it keeps reallocating memory. Just doing buildkernel gives almost 150k calls to free coming from unr. This also gets rid of the hack which tried to postpone TID reuse. Reviewed by: kib, markj Tested by: pho Differential Revision: https://reviews.freebsd.org/D27101 Modified: head/sys/kern/kern_thread.c Modified: head/sys/kern/kern_thread.c ============================================================================== --- head/sys/kern/kern_thread.c Mon Nov 9 23:04:30 2020 (r367536) +++ head/sys/kern/kern_thread.c Mon Nov 9 23:05:28 2020 (r367537) @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -138,9 +139,8 @@ static int thread_unsuspend_one(struct thread *td, str #define TID_BUFFER_SIZE 1024 struct mtx tid_lock; -static struct unrhdr *tid_unrhdr; -static lwpid_t tid_buffer[TID_BUFFER_SIZE]; -static int tid_head, tid_tail; +bitstr_t *tid_bitmap; + static MALLOC_DEFINE(M_TIDHASH, "tidhash", "thread hash"); static int maxthread; @@ -163,14 +163,14 @@ tid_alloc(void) { static struct timeval lastfail; static int curfail; - int nthreads_new; + static lwpid_t trytid; lwpid_t tid; - nthreads_new = atomic_fetchadd_int(&nthreads, 1) + 1; - if (nthreads_new >= maxthread - 100) { + mtx_lock(&tid_lock); + if (nthreads + 1 >= maxthread - 100) { if (priv_check_cred(curthread->td_ucred, PRIV_MAXPROC) != 0 || - nthreads_new >= maxthread) { - atomic_subtract_int(&nthreads, 1); + nthreads + 1 >= maxthread) { + mtx_unlock(&tid_lock); if (ppsratecheck(&lastfail, &curfail, 1)) { printf("maxthread limit exceeded by uid %u " "(pid %d); consider increasing kern.maxthread\n", @@ -180,36 +180,40 @@ tid_alloc(void) } } - tid = alloc_unr(tid_unrhdr); - if (tid != -1) - return (tid); - mtx_lock(&tid_lock); - if (tid_head == tid_tail) { - mtx_unlock(&tid_lock); - return (-1); + nthreads++; + /* + * It is an invariant that the bitmap is big enough to hold maxthread + * IDs. If we got to this point there has to be at least one free. + */ + if (trytid >= maxthread) + trytid = 0; + bit_ffc_at(tid_bitmap, trytid, maxthread, &tid); + if (tid == -1) { + KASSERT(trytid != 0, ("unexpectedly ran out of IDs")); + trytid = 0; + bit_ffc_at(tid_bitmap, trytid, maxthread, &tid); + KASSERT(tid != -1, ("unexpectedly ran out of IDs")); } - tid = tid_buffer[tid_head]; - tid_head = (tid_head + 1) % TID_BUFFER_SIZE; + bit_set(tid_bitmap, tid); + trytid++; mtx_unlock(&tid_lock); - return (tid); + return (tid + NO_PID); } static void -tid_free(lwpid_t tid) +tid_free(lwpid_t rtid) { - lwpid_t tmp_tid = -1; + lwpid_t tid; + KASSERT(rtid >= NO_PID, + ("%s: invalid tid %d\n", __func__, rtid)); + tid = rtid - NO_PID; mtx_lock(&tid_lock); - if ((tid_tail + 1) % TID_BUFFER_SIZE == tid_head) { - tmp_tid = tid_buffer[tid_head]; - tid_head = (tid_head + 1) % TID_BUFFER_SIZE; - } - tid_buffer[tid_tail] = tid; - tid_tail = (tid_tail + 1) % TID_BUFFER_SIZE; + KASSERT(bit_test(tid_bitmap, tid) != 0, + ("thread ID %d not allocated\n", rtid)); + bit_clear(tid_bitmap, tid); + nthreads--; mtx_unlock(&tid_lock); - if (tmp_tid != -1) - free_unr(tid_unrhdr, tmp_tid); - atomic_subtract_int(&nthreads, 1); } /* @@ -371,12 +375,7 @@ threadinit(void) } mtx_init(&tid_lock, "TID lock", NULL, MTX_DEF); - - /* - * pid_max cannot be greater than PID_MAX. - * leave one number for thread0. - */ - tid_unrhdr = new_unrhdr(PID_MAX + 2, INT_MAX, &tid_lock); + tid_bitmap = bit_alloc(maxthread, M_TIDHASH, M_WAITOK); flags = UMA_ZONE_NOFREE; #ifdef __aarch64__