From owner-svn-src-projects@FreeBSD.ORG Sat Aug 24 08:06:58 2013 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTP id 55358FA; Sat, 24 Aug 2013 08:06:58 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 334682154; Sat, 24 Aug 2013 08:06:58 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r7O86wLe032135; Sat, 24 Aug 2013 08:06:58 GMT (envelope-from mav@svn.freebsd.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r7O86vTL032134; Sat, 24 Aug 2013 08:06:57 GMT (envelope-from mav@svn.freebsd.org) Message-Id: <201308240806.r7O86vTL032134@svn.freebsd.org> From: Alexander Motin Date: Sat, 24 Aug 2013 08:06:57 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r254763 - projects/camlock/sys/kern X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 24 Aug 2013 08:06:58 -0000 Author: mav Date: Sat Aug 24 08:06:57 2013 New Revision: 254763 URL: http://svnweb.freebsd.org/changeset/base/254763 Log: Move tq_enqueue() call out of the queue lock for known handlers (actually I have found no others in the base system). This reduces queue lock hold time and congestion spinning under active multithreaded enqueuing. According to hwpmc, for ZFS benchmark doing 100K disk IOPS this moved 10% of CPU time spent on lock spinning from this lock to UMA locks. Hope that side also can be addressed somehow sometimes. Modified: projects/camlock/sys/kern/subr_taskqueue.c Modified: projects/camlock/sys/kern/subr_taskqueue.c ============================================================================== --- projects/camlock/sys/kern/subr_taskqueue.c Sat Aug 24 07:19:57 2013 (r254762) +++ projects/camlock/sys/kern/subr_taskqueue.c Sat Aug 24 08:06:57 2013 (r254763) @@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$"); static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues"); static void *taskqueue_giant_ih; static void *taskqueue_ih; +static void taskqueue_fast_enqueue(void *); +static void taskqueue_swi_enqueue(void *); +static void taskqueue_swi_giant_enqueue(void *); struct taskqueue_busy { struct task *tb_running; @@ -69,6 +72,7 @@ struct taskqueue { #define TQ_FLAGS_ACTIVE (1 << 0) #define TQ_FLAGS_BLOCKED (1 << 1) +#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) #define DT_CALLOUT_ARMED (1 << 0) @@ -96,7 +100,8 @@ _timeout_task_init(struct taskqueue *que { TASK_INIT(&timeout_task->t, priority, func, context); - callout_init_mtx(&timeout_task->c, &queue->tq_mutex, 0); + callout_init_mtx(&timeout_task->c, &queue->tq_mutex, + CALLOUT_RETURNUNLOCKED); timeout_task->q = queue; timeout_task->f = 0; } @@ -127,6 +132,11 @@ _taskqueue_create(const char *name __unu queue->tq_context = context; queue->tq_spin = (mtxflags & MTX_SPIN) != 0; queue->tq_flags |= TQ_FLAGS_ACTIVE; + if (enqueue == taskqueue_fast_enqueue || + enqueue == taskqueue_swi_enqueue || + enqueue == taskqueue_swi_giant_enqueue || + enqueue == taskqueue_thread_enqueue) + queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE; mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags); return queue; @@ -196,6 +206,7 @@ taskqueue_enqueue_locked(struct taskqueu if (task->ta_pending) { if (task->ta_pending < USHRT_MAX) task->ta_pending++; + TQ_UNLOCK(queue); return (0); } @@ -219,8 +230,12 @@ taskqueue_enqueue_locked(struct taskqueu } task->ta_pending = 1; + if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) != 0) + TQ_UNLOCK(queue); if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) queue->tq_enqueue(queue->tq_context); + if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) == 0) + TQ_UNLOCK(queue); return (0); } @@ -231,7 +246,6 @@ taskqueue_enqueue(struct taskqueue *queu TQ_LOCK(queue); res = taskqueue_enqueue_locked(queue, task); - TQ_UNLOCK(queue); return (res); } @@ -277,8 +291,8 @@ taskqueue_enqueue_timeout(struct taskque callout_reset(&timeout_task->c, ticks, taskqueue_timeout_func, timeout_task); } + TQ_UNLOCK(queue); } - TQ_UNLOCK(queue); return (res); } @@ -563,7 +577,6 @@ taskqueue_thread_enqueue(void *context) tqp = context; tq = *tqp; - TQ_ASSERT_LOCKED(tq); wakeup_one(tq); }