From owner-svn-src-all@FreeBSD.ORG Wed Oct 16 09:52:59 2013 Return-Path: Delivered-To: svn-src-all@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) (using TLSv1 with cipher ADH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTP id AF578326; Wed, 16 Oct 2013 09:52:59 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mx1.freebsd.org (Postfix) with ESMTPS id 8D2EC2FE4; Wed, 16 Oct 2013 09:52:59 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.7/8.14.7) with ESMTP id r9G9qxnt019497; Wed, 16 Oct 2013 09:52:59 GMT (envelope-from mav@svn.freebsd.org) Received: (from mav@localhost) by svn.freebsd.org (8.14.7/8.14.5/Submit) id r9G9qxmD019496; Wed, 16 Oct 2013 09:52:59 GMT (envelope-from mav@svn.freebsd.org) Message-Id: <201310160952.r9G9qxmD019496@svn.freebsd.org> From: Alexander Motin Date: Wed, 16 Oct 2013 09:52:59 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r256613 - head/sys/kern X-SVN-Group: head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.14 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 16 Oct 2013 09:52:59 -0000 Author: mav Date: Wed Oct 16 09:52:59 2013 New Revision: 256613 URL: http://svnweb.freebsd.org/changeset/base/256613 Log: MFprojects/camlock r254763: Move tq_enqueue() call out of the queue lock for known handlers (actually I have found no others in the base system). This reduces queue lock hold time and congestion spinning under active multithreaded enqueuing. Modified: head/sys/kern/subr_taskqueue.c Modified: head/sys/kern/subr_taskqueue.c ============================================================================== --- head/sys/kern/subr_taskqueue.c Wed Oct 16 09:48:23 2013 (r256612) +++ head/sys/kern/subr_taskqueue.c Wed Oct 16 09:52:59 2013 (r256613) @@ -47,6 +47,9 @@ __FBSDID("$FreeBSD$"); static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues"); static void *taskqueue_giant_ih; static void *taskqueue_ih; +static void taskqueue_fast_enqueue(void *); +static void taskqueue_swi_enqueue(void *); +static void taskqueue_swi_giant_enqueue(void *); struct taskqueue_busy { struct task *tb_running; @@ -70,6 +73,7 @@ struct taskqueue { #define TQ_FLAGS_ACTIVE (1 << 0) #define TQ_FLAGS_BLOCKED (1 << 1) +#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2) #define DT_CALLOUT_ARMED (1 << 0) @@ -97,7 +101,8 @@ _timeout_task_init(struct taskqueue *que { TASK_INIT(&timeout_task->t, priority, func, context); - callout_init_mtx(&timeout_task->c, &queue->tq_mutex, 0); + callout_init_mtx(&timeout_task->c, &queue->tq_mutex, + CALLOUT_RETURNUNLOCKED); timeout_task->q = queue; timeout_task->f = 0; } @@ -128,6 +133,11 @@ _taskqueue_create(const char *name __unu queue->tq_context = context; queue->tq_spin = (mtxflags & MTX_SPIN) != 0; queue->tq_flags |= TQ_FLAGS_ACTIVE; + if (enqueue == taskqueue_fast_enqueue || + enqueue == taskqueue_swi_enqueue || + enqueue == taskqueue_swi_giant_enqueue || + enqueue == taskqueue_thread_enqueue) + queue->tq_flags |= TQ_FLAGS_UNLOCKED_ENQUEUE; mtx_init(&queue->tq_mutex, mtxname, NULL, mtxflags); return queue; @@ -197,6 +207,7 @@ taskqueue_enqueue_locked(struct taskqueu if (task->ta_pending) { if (task->ta_pending < USHRT_MAX) task->ta_pending++; + TQ_UNLOCK(queue); return (0); } @@ -220,8 +231,12 @@ taskqueue_enqueue_locked(struct taskqueu } task->ta_pending = 1; + if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) != 0) + TQ_UNLOCK(queue); if ((queue->tq_flags & TQ_FLAGS_BLOCKED) == 0) queue->tq_enqueue(queue->tq_context); + if ((queue->tq_flags & TQ_FLAGS_UNLOCKED_ENQUEUE) == 0) + TQ_UNLOCK(queue); return (0); } @@ -232,7 +247,6 @@ taskqueue_enqueue(struct taskqueue *queu TQ_LOCK(queue); res = taskqueue_enqueue_locked(queue, task); - TQ_UNLOCK(queue); return (res); } @@ -278,8 +292,8 @@ taskqueue_enqueue_timeout(struct taskque callout_reset(&timeout_task->c, ticks, taskqueue_timeout_func, timeout_task); } + TQ_UNLOCK(queue); } - TQ_UNLOCK(queue); return (res); } @@ -566,7 +580,6 @@ taskqueue_thread_enqueue(void *context) tqp = context; tq = *tqp; - TQ_ASSERT_LOCKED(tq); wakeup_one(tq); }