From owner-svn-src-user@FreeBSD.ORG Fri Nov 21 00:57:50 2008 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 7DE021065672; Fri, 21 Nov 2008 00:57:50 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 683348FC1A; Fri, 21 Nov 2008 00:57:50 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id mAL0voMX068442; Fri, 21 Nov 2008 00:57:50 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id mAL0voKZ068441; Fri, 21 Nov 2008 00:57:50 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200811210057.mAL0voKZ068441@svn.freebsd.org> From: Kip Macy Date: Fri, 21 Nov 2008 00:57:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r185136 - user/kmacy/HEAD_fast_multi_xmit/sys/sys X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 21 Nov 2008 00:57:50 -0000 Author: kmacy Date: Fri Nov 21 00:57:50 2008 New Revision: 185136 URL: http://svn.freebsd.org/changeset/base/185136 Log: - Add single consumer variant of dequeue for cases where the consumer is already protected by a lock - move enqueue/dequeue to critical sections to avoid starvation by preemption Modified: user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h Modified: user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h ============================================================================== --- user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h Fri Nov 21 00:48:57 2008 (r185135) +++ user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h Fri Nov 21 00:57:50 2008 (r185136) @@ -86,14 +86,17 @@ buf_ring_enqueue(struct buf_ring *br, vo uint32_t cons_tail; int success; + critical_enter(); do { prod_head = br->br_prod_head; cons_tail = br->br_cons_tail; prod_next = (prod_head + 1) & br->br_prod_mask; - if (prod_next == cons_tail) + if (prod_next == cons_tail) { + critical_exit(); return (ENOSPC); + } success = atomic_cmpset_int(&br->br_prod_head, prod_head, prod_next); @@ -107,32 +110,38 @@ buf_ring_enqueue(struct buf_ring *br, vo /* * If there are other enqueues in progress * that preceeded us, we need to wait for them - * to complete - this is only a problem if they - * have been preempted + * to complete */ while (br->br_prod_tail != prod_head) cpu_spinwait(); br->br_prod_tail = prod_next; - + critical_exit(); return (0); } +/* + * multi-consumer safe dequeue + * + */ static __inline void * -buf_ring_dequeue(struct buf_ring *br) +buf_ring_dequeue_mc(struct buf_ring *br) { uint32_t cons_head, cons_next; uint32_t prod_tail; void *buf; int success; - + + critical_enter(); do { cons_head = br->br_cons_head; prod_tail = br->br_prod_tail; cons_next = (cons_head + 1) & br->br_cons_mask; - if (cons_head == prod_tail) + if (cons_head == prod_tail) { + critical_exit(); return (NULL); + } success = atomic_cmpset_int(&br->br_cons_head, cons_head, cons_next); @@ -147,14 +156,53 @@ buf_ring_dequeue(struct buf_ring *br) /* * If there are other dequeues in progress * that preceeded us, we need to wait for them - * to complete - this is only a problem if they - * have been preempted + * to complete */ while (br->br_cons_tail != cons_head) cpu_spinwait(); br->br_cons_tail = cons_next; + mb(); + critical_exit(); + + return (buf); +} + +/* + * Single-Consumer dequeue for uses where dequeue + * is protected by a lock + */ +static __inline void * +buf_ring_dequeue_sc(struct buf_ring *br) +{ + uint32_t cons_head, cons_next; + uint32_t prod_tail; + void *buf; + + critical_enter(); + cons_head = br->br_cons_head; + prod_tail = br->br_prod_tail; + + cons_next = (cons_head + 1) & br->br_cons_mask; + + if (cons_head == prod_tail) { + critical_exit(); + return (NULL); + } + + br->br_cons_head = cons_next; + buf = br->br_ring[cons_head]; +#ifdef INVARIANTS + br->br_ring[cons_head] = NULL; +#endif + mb(); + KASSERT(br->br_cons_tail == cons_head, + ("inconsistent list cons_tail=%d cons_head=%d", + br->br_cons_tail, cons_head)); + br->br_cons_tail = cons_next; + mb(); + critical_exit(); return (buf); }