Date: Thu, 20 Nov 2008 10:47:21 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r185126 - in user/kmacy/HEAD_fast_multi_xmit/sys: kern sys Message-ID: <200811201047.mAKAlLij049286@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Thu Nov 20 10:47:21 2008 New Revision: 185126 URL: http://svn.freebsd.org/changeset/base/185126 Log: Add lock-less ring buffer as primitive for replacing ifq Added: user/kmacy/HEAD_fast_multi_xmit/sys/kern/subr_bufr.c (contents, props changed) user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h (contents, props changed) Added: user/kmacy/HEAD_fast_multi_xmit/sys/kern/subr_bufr.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/HEAD_fast_multi_xmit/sys/kern/subr_bufr.c Thu Nov 20 10:47:21 2008 (r185126) @@ -0,0 +1,66 @@ +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * + ***************************************************************************/ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/malloc.h> +#include <sys/ktr.h> +#include <sys/buf_ring.h> + + +struct buf_ring * +buf_ring_alloc(int count, struct malloc_type *type, int flags) +{ + struct buf_ring *br; + + KASSERT(powerof2(count), ("buf ring must be size power of 2")); + + br = malloc(sizeof(struct buf_ring) + count*sizeof(caddr_t), + type, flags|M_ZERO); + if (br == NULL) + return (NULL); + + br->br_prod_size = br->br_cons_size = count; + br->br_prod_mask = br->br_cons_mask = count-1; + br->br_prod_head = br->br_cons_head = 0; + br->br_prod_tail = br->br_cons_tail = 0; + + return (br); +} + +void +buf_ring_free(struct buf_ring *br, struct malloc_type *type) +{ + free(br, type); +} Added: user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/HEAD_fast_multi_xmit/sys/sys/buf_ring.h Thu Nov 20 10:47:21 2008 (r185126) @@ -0,0 +1,195 @@ +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + ***************************************************************************/ + +#ifndef _SYS_BUF_RING_H_ +#define _SYS_BUF_RING_H_ + +#include <machine/cpu.h> + +#if defined (__GNUC__) + #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__ || #cpu(x86_64) || defined __x86_64__ + #define mb() __asm__ __volatile__ ("mfence;": : :"memory") + #define wmb() __asm__ __volatile__ ("sfence;": : :"memory") + #define rmb() __asm__ __volatile__ ("lfence;": : :"memory") + #elif #cpu(sparc64) || defined sparc64 || defined __sparcv9 + #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") + #define wmb() mb() + #define rmb() mb() + #elif #cpu(sparc) || defined sparc || defined __sparc__ + #define mb() __asm__ __volatile__ ("stbar;": : :"memory") + #define wmb() mb() + #define rmb() mb() +#else + #define wmb() mb() + #define rmb() mb() + #define mb() /* XXX just to make this compile */ + #endif +#else + #error "unknown compiler" +#endif + +struct buf_ring { + volatile uint32_t br_prod_head; + volatile uint32_t br_prod_tail; + int br_prod_size; + int br_prod_mask; + /* + * Pad out to next L2 cache line + */ + uint64_t _pad0[14]; + + volatile uint32_t br_cons_head; + volatile uint32_t br_cons_tail; + int br_cons_size; + int br_cons_mask; + + /* + * Pad out to next L2 cache line + */ + uint64_t _pad1[14]; + void *br_ring[0]; +}; + + +static __inline int +buf_ring_enqueue(struct buf_ring *br, void *buf) +{ + uint32_t prod_head, prod_next; + uint32_t cons_tail; + int success; + + do { + prod_head = br->br_prod_head; + cons_tail = br->br_cons_tail; + + prod_next = (prod_head + 1) & br->br_prod_mask; + + if (prod_next == cons_tail) + return (ENOSPC); + + success = atomic_cmpset_int(&br->br_prod_head, prod_head, + prod_next); + } while (success == 0); + + KASSERT(br->br_ring[prod_head] == NULL, ("dangling value in enqueue")); + + br->br_ring[prod_head] = buf; + wmb(); + + /* + * If there are other enqueues in progress + * that preceeded us, we need to wait for them + * to complete - this is only a problem if they + * have been preempted + */ + while (br->br_prod_tail != prod_head) + cpu_spinwait(); + br->br_prod_tail = prod_next; + + return (0); +} + +static __inline void * +buf_ring_dequeue(struct buf_ring *br) +{ + uint32_t cons_head, cons_next; + uint32_t prod_tail; + void *buf; + int success; + + do { + cons_head = br->br_cons_head; + prod_tail = br->br_prod_tail; + + cons_next = (cons_head + 1) & br->br_cons_mask; + + if (cons_head == prod_tail) + return (NULL); + + success = atomic_cmpset_int(&br->br_cons_head, cons_head, + cons_next); + } while (success == 0); + + buf = br->br_ring[cons_head]; +#ifdef INVARIANTS + br->br_ring[cons_head] = NULL; +#endif + mb(); + + /* + * If there are other dequeues in progress + * that preceeded us, we need to wait for them + * to complete - this is only a problem if they + * have been preempted + */ + while (br->br_cons_tail != cons_head) + cpu_spinwait(); + + br->br_cons_tail = cons_next; + + return (buf); +} + +static __inline void * +buf_ring_peek(struct buf_ring *br) +{ + + return (br->br_ring[br->br_cons_tail]); +} + +static __inline int +buf_ring_full(struct buf_ring *br) +{ + + return (((br->br_prod_head + 1) & br->br_prod_mask) == br->br_cons_tail); +} + +static __inline int +buf_ring_empty(struct buf_ring *br) +{ + + return (br->br_cons_head == br->br_prod_tail); +} + +static __inline int +buf_ring_count(struct buf_ring *br) +{ + + return ((br->br_prod_size + br->br_prod_tail - br->br_cons_tail) + & br->br_prod_mask); +} + +struct buf_ring *buf_ring_alloc(int count, struct malloc_type *type, int flags); +void buf_ring_free(struct buf_ring *br, struct malloc_type *type); + + + +#endif
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200811201047.mAKAlLij049286>