From nobody Thu Oct 14 23:14:23 2021 X-Original-To: dev-commits-src-main@mlmmj.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mlmmj.nyi.freebsd.org (Postfix) with ESMTP id AB63717F76E7; Thu, 14 Oct 2021 23:14:23 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256 client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "R3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 4HVlbM4QSxz3mVG; Thu, 14 Oct 2021 23:14:23 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org (gitrepo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:5]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 77E731BE5C; Thu, 14 Oct 2021 23:14:23 +0000 (UTC) (envelope-from git@FreeBSD.org) Received: from gitrepo.freebsd.org ([127.0.1.44]) by gitrepo.freebsd.org (8.16.1/8.16.1) with ESMTP id 19ENENba017471; Thu, 14 Oct 2021 23:14:23 GMT (envelope-from git@gitrepo.freebsd.org) Received: (from git@localhost) by gitrepo.freebsd.org (8.16.1/8.16.1/Submit) id 19ENENxk017470; Thu, 14 Oct 2021 23:14:23 GMT (envelope-from git) Date: Thu, 14 Oct 2021 23:14:23 GMT Message-Id: <202110142314.19ENENxk017470@gitrepo.freebsd.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org From: John Baldwin Subject: git: a72ee355646c - main - ktls: Defer creation of threads and zones until first use. List-Id: Commit messages for the main branch of the src repository List-Archive: https://lists.freebsd.org/archives/dev-commits-src-main List-Help: List-Post: List-Subscribe: List-Unsubscribe: Sender: owner-dev-commits-src-main@freebsd.org X-BeenThere: dev-commits-src-main@freebsd.org MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit X-Git-Committer: jhb X-Git-Repository: src X-Git-Refname: refs/heads/main X-Git-Reftype: branch X-Git-Commit: a72ee355646c4a379f55ac32d7d8dfe6c0d366f9 Auto-Submitted: auto-generated X-ThisMailContainsUnwantedMimeParts: N The branch main has been updated by jhb: URL: https://cgit.FreeBSD.org/src/commit/?id=a72ee355646c4a379f55ac32d7d8dfe6c0d366f9 commit a72ee355646c4a379f55ac32d7d8dfe6c0d366f9 Author: John Baldwin AuthorDate: 2021-10-14 22:48:34 +0000 Commit: John Baldwin CommitDate: 2021-10-14 22:48:34 +0000 ktls: Defer creation of threads and zones until first use. Run ktls_init() when the first KTLS session is created rather than unconditionally during boot. This avoids creating unused threads and allocating unused resources on systems which do not use KTLS. Reviewed by: gallatin, markj Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D32487 --- sys/kern/uipc_ktls.c | 173 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 115 insertions(+), 58 deletions(-) diff --git a/sys/kern/uipc_ktls.c b/sys/kern/uipc_ktls.c index 1e778dbf113a..bde9fca97f50 100644 --- a/sys/kern/uipc_ktls.c +++ b/sys/kern/uipc_ktls.c @@ -109,6 +109,9 @@ static struct proc *ktls_proc; static uma_zone_t ktls_session_zone; static uma_zone_t ktls_buffer_zone; static uint16_t ktls_cpuid_lookup[MAXCPU]; +static int ktls_init_state; +static struct sx ktls_init_lock; +SX_SYSINIT(ktls_init_lock, &ktls_init_lock, "ktls init"); SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, "Kernel TLS offload"); @@ -379,12 +382,11 @@ ktls_free_mext_contig(struct mbuf *m) uma_zfree(ktls_buffer_zone, (void *)PHYS_TO_DMAP(m->m_epg_pa[0])); } -static void -ktls_init(void *dummy __unused) +static int +ktls_init(void) { struct thread *td; struct pcpu *pc; - cpuset_t mask; int count, domain, error, i; ktls_wq = malloc(sizeof(*ktls_wq) * (mp_maxid + 1), M_KTLS, @@ -410,36 +412,40 @@ ktls_init(void *dummy __unused) STAILQ_INIT(&ktls_wq[i].m_head); STAILQ_INIT(&ktls_wq[i].so_head); mtx_init(&ktls_wq[i].mtx, "ktls work queue", NULL, MTX_DEF); - error = kproc_kthread_add(ktls_work_thread, &ktls_wq[i], - &ktls_proc, &td, 0, 0, "KTLS", "thr_%d", i); - if (error) - panic("Can't add KTLS thread %d error %d", i, error); - - /* - * Bind threads to cores. If ktls_bind_threads is > - * 1, then we bind to the NUMA domain. - */ - if (ktls_bind_threads) { - if (ktls_bind_threads > 1) { - pc = pcpu_find(i); - domain = pc->pc_domain; - CPU_COPY(&cpuset_domain[domain], &mask); - count = ktls_domains[domain].count; - ktls_domains[domain].cpu[count] = i; - ktls_domains[domain].count++; - } else { - CPU_SETOF(i, &mask); - } - error = cpuset_setthread(td->td_tid, &mask); - if (error) - panic( - "Unable to bind KTLS thread for CPU %d error %d", - i, error); + if (ktls_bind_threads > 1) { + pc = pcpu_find(i); + domain = pc->pc_domain; + count = ktls_domains[domain].count; + ktls_domains[domain].cpu[count] = i; + ktls_domains[domain].count++; } ktls_cpuid_lookup[ktls_number_threads] = i; ktls_number_threads++; } + /* + * If we somehow have an empty domain, fall back to choosing + * among all KTLS threads. + */ + if (ktls_bind_threads > 1) { + for (i = 0; i < vm_ndomains; i++) { + if (ktls_domains[i].count == 0) { + ktls_bind_threads = 1; + break; + } + } + } + + /* Start kthreads for each workqueue. */ + CPU_FOREACH(i) { + error = kproc_kthread_add(ktls_work_thread, &ktls_wq[i], + &ktls_proc, &td, 0, 0, "KTLS", "thr_%d", i); + if (error) { + printf("Can't add KTLS thread %d error %d\n", i, error); + return (error); + } + } + /* * Start an allocation thread per-domain to perform blocking allocations * of 16k physically contiguous TLS crypto destination buffers. @@ -454,35 +460,46 @@ ktls_init(void *dummy __unused) &ktls_domains[domain], &ktls_proc, &ktls_domains[domain].alloc_td.td, 0, 0, "KTLS", "alloc_%d", domain); - if (error) - panic("Can't add KTLS alloc thread %d error %d", + if (error) { + printf("Can't add KTLS alloc thread %d error %d\n", domain, error); - CPU_COPY(&cpuset_domain[domain], &mask); - error = cpuset_setthread(ktls_domains[domain].alloc_td.td->td_tid, - &mask); - if (error) - panic("Unable to bind KTLS alloc %d error %d", - domain, error); - } - } - - /* - * If we somehow have an empty domain, fall back to choosing - * among all KTLS threads. - */ - if (ktls_bind_threads > 1) { - for (i = 0; i < vm_ndomains; i++) { - if (ktls_domains[i].count == 0) { - ktls_bind_threads = 1; - break; + return (error); } } } if (bootverbose) printf("KTLS: Initialized %d threads\n", ktls_number_threads); + return (0); +} + +static int +ktls_start_kthreads(void) +{ + int error, state; + +start: + state = atomic_load_acq_int(&ktls_init_state); + if (__predict_true(state > 0)) + return (0); + if (state < 0) + return (ENXIO); + + sx_xlock(&ktls_init_lock); + if (ktls_init_state != 0) { + sx_xunlock(&ktls_init_lock); + goto start; + } + + error = ktls_init(); + if (error == 0) + state = 1; + else + state = -1; + atomic_store_rel_int(&ktls_init_state, state); + sx_xunlock(&ktls_init_lock); + return (error); } -SYSINIT(ktls, SI_SUB_SMP + 1, SI_ORDER_ANY, ktls_init, NULL); #if defined(INET) || defined(INET6) static int @@ -583,6 +600,10 @@ ktls_create_session(struct socket *so, struct tls_enable *en, return (EINVAL); } + error = ktls_start_kthreads(); + if (error != 0) + return (error); + tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO); counter_u64_add(ktls_offload_active, 1); @@ -2457,6 +2478,18 @@ ktls_encrypt_async(struct ktls_wq *wq, struct mbuf *top) CURVNET_RESTORE(); } +static int +ktls_bind_domain(int domain) +{ + int error; + + error = cpuset_setthread(curthread->td_tid, &cpuset_domain[domain]); + if (error != 0) + return (error); + curthread->td_domain.dr_policy = DOMAINSET_PREF(domain); + return (0); +} + static void ktls_alloc_thread(void *ctx) { @@ -2465,14 +2498,16 @@ ktls_alloc_thread(void *ctx) void **buf; struct sysctl_oid *oid; char name[80]; - int i, nbufs; + int domain, error, i, nbufs; - curthread->td_domain.dr_policy = - DOMAINSET_PREF(PCPU_GET(domain)); - snprintf(name, sizeof(name), "domain%d", PCPU_GET(domain)); + domain = ktls_domain - ktls_domains; if (bootverbose) - printf("Starting KTLS alloc thread for domain %d\n", - PCPU_GET(domain)); + printf("Starting KTLS alloc thread for domain %d\n", domain); + error = ktls_bind_domain(domain); + if (error) + printf("Unable to bind KTLS alloc thread for domain %d: error %d\n", + domain, error); + snprintf(name, sizeof(name), "domain%d", domain); oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_kern_ipc_tls), OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, "allocs", @@ -2527,10 +2562,32 @@ ktls_work_thread(void *ctx) struct socket *so, *son; STAILQ_HEAD(, mbuf) local_m_head; STAILQ_HEAD(, socket) local_so_head; + int cpu; - if (ktls_bind_threads > 1) { - curthread->td_domain.dr_policy = - DOMAINSET_PREF(PCPU_GET(domain)); + cpu = wq - ktls_wq; + if (bootverbose) + printf("Starting KTLS worker thread for CPU %d\n", cpu); + + /* + * Bind to a core. If ktls_bind_threads is > 1, then + * we bind to the NUMA domain instead. + */ + if (ktls_bind_threads) { + int error; + + if (ktls_bind_threads > 1) { + struct pcpu *pc = pcpu_find(cpu); + + error = ktls_bind_domain(pc->pc_domain); + } else { + cpuset_t mask; + + CPU_SETOF(cpu, &mask); + error = cpuset_setthread(curthread->td_tid, &mask); + } + if (error) + printf("Unable to bind KTLS worker thread for CPU %d: error %d\n", + cpu, error); } #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__) fpu_kern_thread(0);