From owner-svn-src-projects@FreeBSD.ORG Sat Aug 24 22:42:20 2013 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTP id 4D3D05E8; Sat, 24 Aug 2013 22:42:20 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 2B1C32870; Sat, 24 Aug 2013 22:42:20 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r7OMgKYV021793; Sat, 24 Aug 2013 22:42:20 GMT (envelope-from mav@svn.freebsd.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r7OMgJrH021790; Sat, 24 Aug 2013 22:42:19 GMT (envelope-from mav@svn.freebsd.org) Message-Id: <201308242242.r7OMgJrH021790@svn.freebsd.org> From: Alexander Motin Date: Sat, 24 Aug 2013 22:42:19 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r254816 - in projects/camlock/sys/cddl: compat/opensolaris/kern contrib/opensolaris/uts/common/sys X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 24 Aug 2013 22:42:20 -0000 Author: mav Date: Sat Aug 24 22:42:19 2013 New Revision: 254816 URL: http://svnweb.freebsd.org/changeset/base/254816 Log: Make taskqueue wrapper used for ZFS more SMP-scalable. For some purposes ZFS requests to create taskqueues with number of threads equal to number of CPUs. As result, on 24-core system we may get situation when interoperation between 5 CAM completion threads and 24 ZFS taskqueue threads protected with single global lock. When number of IOPS reaches hundreds of thousands, that causes significant lock congestion. To workaround that, create several FreeBSD taskqueues to emulate single Solaris taskqueue and distribute tasks between them in round-robin fashion. Using 4 taskqueues of 6 threads for the 24-core system almost completely removed spinning on these locks. Modified: projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Modified: projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c ============================================================================== --- projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Sat Aug 24 21:30:35 2013 (r254815) +++ projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c Sat Aug 24 22:42:19 2013 (r254816) @@ -66,14 +66,26 @@ taskq_create(const char *name, int nthre int maxalloc __unused, uint_t flags) { taskq_t *tq; + int i, nqueues; if ((flags & TASKQ_THREADS_CPU_PCT) != 0) nthreads = MAX((mp_ncpus * nthreads) / 100, 1); + nqueues = MAX((nthreads + 4) / 6, 1); + nthreads = MAX((nthreads + nqueues / 2) / nqueues, 1); - tq = kmem_alloc(sizeof(*tq), KM_SLEEP); - tq->tq_queue = taskqueue_create(name, M_WAITOK, taskqueue_thread_enqueue, - &tq->tq_queue); - (void) taskqueue_start_threads(&tq->tq_queue, nthreads, pri, "%s", name); + tq = kmem_alloc(sizeof(*tq) + sizeof(tq->tq_queue[0]) * nqueues, KM_SLEEP); + tq->tq_num = nqueues; + tq->tq_last = 0; + for (i = 0; i < nqueues; i++) { + tq->tq_queue[i] = taskqueue_create(name, M_WAITOK, + taskqueue_thread_enqueue, &tq->tq_queue[i]); + if (nqueues == 1) + (void) taskqueue_start_threads(&tq->tq_queue[i], + nthreads, pri, "%s", name); + else + (void) taskqueue_start_threads(&tq->tq_queue[i], + nthreads, pri, "%s_%d", name, i); + } return ((taskq_t *)tq); } @@ -89,16 +101,22 @@ taskq_create_proc(const char *name, int void taskq_destroy(taskq_t *tq) { + int i; - taskqueue_free(tq->tq_queue); - kmem_free(tq, sizeof(*tq)); + for (i = 0; i < tq->tq_num; i++) + taskqueue_free(tq->tq_queue[i]); + kmem_free(tq, sizeof(*tq) + sizeof(tq->tq_queue[0]) * tq->tq_num); } int taskq_member(taskq_t *tq, kthread_t *thread) { + int i, j; - return (taskqueue_member(tq->tq_queue, thread)); + for (i = 0; i < tq->tq_num; i++) + if (taskqueue_member(tq->tq_queue[i], thread)) + return (1); + return (0); } static void @@ -115,7 +133,7 @@ taskqid_t taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) { struct ostask *task; - int mflag, prio; + int i, mflag, prio; if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP) mflag = M_WAITOK; @@ -135,7 +153,11 @@ taskq_dispatch(taskq_t *tq, task_func_t task->ost_arg = arg; TASK_INIT(&task->ost_task, prio, taskq_run, task); - taskqueue_enqueue(tq->tq_queue, &task->ost_task); + if (tq->tq_num > 1) + i = atomic_fetchadd_int(&tq->tq_last, 1) % tq->tq_num; + else + i = 0; + taskqueue_enqueue(tq->tq_queue[i], &task->ost_task); return ((taskqid_t)(void *)task); } @@ -154,7 +176,7 @@ taskqid_t taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg, u_int flags, struct ostask *task) { - int prio; + int i, prio; /* * If TQ_FRONT is given, we want higher priority for this task, so it @@ -166,7 +188,11 @@ taskq_dispatch_safe(taskq_t *tq, task_fu task->ost_arg = arg; TASK_INIT(&task->ost_task, prio, taskq_run_safe, task); - taskqueue_enqueue(tq->tq_queue, &task->ost_task); + if (tq->tq_num > 1) + i = atomic_fetchadd_int(&tq->tq_last, 1) % tq->tq_num; + else + i = 0; + taskqueue_enqueue(tq->tq_queue[i], &task->ost_task); return ((taskqid_t)(void *)task); } Modified: projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h ============================================================================== --- projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Sat Aug 24 21:30:35 2013 (r254815) +++ projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h Sat Aug 24 22:42:19 2013 (r254816) @@ -38,7 +38,9 @@ extern "C" { struct taskqueue; struct taskq { - struct taskqueue *tq_queue; + int tq_num; + int tq_last; + struct taskqueue *tq_queue[]; }; typedef struct taskq taskq_t;