Date: Thu, 12 Apr 2012 11:27:09 +0000 (UTC) From: Luigi Rizzo <luigi@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r234174 - head/sys/dev/netmap Message-ID: <201204121127.q3CBR9VO026345@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: luigi Date: Thu Apr 12 11:27:09 2012 New Revision: 234174 URL: http://svn.freebsd.org/changeset/base/234174 Log: Some code restructuring to bring the memory allocator out of netmap.c and make it easier to replace it with a different implementation. On passing, also fix indentation. NOTE: I know that #include "foo.c" is ugly, but the alternative (add another entry to sys/conf/files, add a separate header with structs and prototypes, and expose functions that are meant to be private) looks even worse to me. We need a more modular way to specify dependencies and build options. Added: head/sys/dev/netmap/netmap_mem1.c (contents, props changed) Modified: head/sys/dev/netmap/netmap.c head/sys/dev/netmap/netmap_kern.h Modified: head/sys/dev/netmap/netmap.c ============================================================================== --- head/sys/dev/netmap/netmap.c Thu Apr 12 11:23:24 2012 (r234173) +++ head/sys/dev/netmap/netmap.c Thu Apr 12 11:27:09 2012 (r234174) @@ -116,507 +116,13 @@ SYSCTL_INT(_dev_netmap, OID_AUTO, no_pen CTLFLAG_RW, &netmap_no_pendintr, 0, "Always look for new received packets."); - -/*----- memory allocator -----------------*/ -/* - * Here we have the low level routines for memory allocator - * and its primary users. - */ - -/* - * Default amount of memory pre-allocated by the module. - * We start with a large size and then shrink our demand - * according to what is avalable when the module is loaded. - * At the moment the block is contiguous, but we can easily - * restrict our demand to smaller units (16..64k) - */ -#define NETMAP_MEMORY_SIZE (64 * 1024 * PAGE_SIZE) -static void * netmap_malloc(size_t size, const char *msg); -static void netmap_free(void *addr, const char *msg); - -#define netmap_if_malloc(len) netmap_malloc(len, "nifp") -#define netmap_if_free(v) netmap_free((v), "nifp") - -#define netmap_ring_malloc(len) netmap_malloc(len, "ring") -#define netmap_free_rings(na) \ - netmap_free((na)->tx_rings[0].ring, "shadow rings"); - -/* - * Allocator for a pool of packet buffers. For each buffer we have - * one entry in the bitmap to signal the state. Allocation scans - * the bitmap, but since this is done only on attach, we are not - * too worried about performance - * XXX if we need to allocate small blocks, a translation - * table is used both for kernel virtual address and physical - * addresses. - */ -struct netmap_buf_pool { - u_int total_buffers; /* total buffers. */ - u_int free; - u_int bufsize; - char *base; /* buffer base address */ - uint32_t *bitmap; /* one bit per buffer, 1 means free */ -}; -struct netmap_buf_pool nm_buf_pool; -SYSCTL_INT(_dev_netmap, OID_AUTO, total_buffers, - CTLFLAG_RD, &nm_buf_pool.total_buffers, 0, "total_buffers"); -SYSCTL_INT(_dev_netmap, OID_AUTO, free_buffers, - CTLFLAG_RD, &nm_buf_pool.free, 0, "free_buffers"); - - - - -/* - * Allocate n buffers from the ring, and fill the slot. - * Buffer 0 is the 'junk' buffer. - */ -static void -netmap_new_bufs(struct netmap_if *nifp __unused, - struct netmap_slot *slot, u_int n) -{ - struct netmap_buf_pool *p = &nm_buf_pool; - uint32_t bi = 0; /* index in the bitmap */ - uint32_t mask, j, i = 0; /* slot counter */ - - if (n > p->free) { - D("only %d out of %d buffers available", i, n); - return; - } - /* termination is guaranteed by p->free */ - while (i < n && p->free > 0) { - uint32_t cur = p->bitmap[bi]; - if (cur == 0) { /* bitmask is fully used */ - bi++; - continue; - } - /* locate a slot */ - for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) ; - p->bitmap[bi] &= ~mask; /* slot in use */ - p->free--; - slot[i].buf_idx = bi*32+j; - slot[i].len = p->bufsize; - slot[i].flags = NS_BUF_CHANGED; - i++; - } - ND("allocated %d buffers, %d available", n, p->free); -} - - -static void -netmap_free_buf(struct netmap_if *nifp __unused, uint32_t i) -{ - struct netmap_buf_pool *p = &nm_buf_pool; - - uint32_t pos, mask; - if (i >= p->total_buffers) { - D("invalid free index %d", i); - return; - } - pos = i / 32; - mask = 1 << (i % 32); - if (p->bitmap[pos] & mask) { - D("slot %d already free", i); - return; - } - p->bitmap[pos] |= mask; - p->free++; -} - - -/* Descriptor of the memory objects handled by our memory allocator. */ -struct netmap_mem_obj { - TAILQ_ENTRY(netmap_mem_obj) nmo_next; /* next object in the - chain. */ - int nmo_used; /* flag set on used memory objects. */ - size_t nmo_size; /* size of the memory area reserved for the - object. */ - void *nmo_data; /* pointer to the memory area. */ -}; - -/* Wrap our memory objects to make them ``chainable``. */ -TAILQ_HEAD(netmap_mem_obj_h, netmap_mem_obj); - - -/* Descriptor of our custom memory allocator. */ -struct netmap_mem_d { - struct mtx nm_mtx; /* lock used to handle the chain of memory - objects. */ - struct netmap_mem_obj_h nm_molist; /* list of memory objects */ - size_t nm_size; /* total amount of memory used for rings etc. */ - size_t nm_totalsize; /* total amount of allocated memory - (the difference is used for buffers) */ - size_t nm_buf_start; /* offset of packet buffers. - This is page-aligned. */ - size_t nm_buf_len; /* total memory for buffers */ - void *nm_buffer; /* pointer to the whole pre-allocated memory - area. */ -}; - -/* Shorthand to compute a netmap interface offset. */ -#define netmap_if_offset(v) \ - ((char *) (v) - (char *) nm_mem->nm_buffer) -/* .. and get a physical address given a memory offset */ -#define netmap_ofstophys(o) \ - (vtophys(nm_mem->nm_buffer) + (o)) - - -/*------ netmap memory allocator -------*/ -/* - * Request for a chunk of memory. - * - * Memory objects are arranged into a list, hence we need to walk this - * list until we find an object with the needed amount of data free. - * This sounds like a completely inefficient implementation, but given - * the fact that data allocation is done once, we can handle it - * flawlessly. - * - * Return NULL on failure. - */ -static void * -netmap_malloc(size_t size, __unused const char *msg) -{ - struct netmap_mem_obj *mem_obj, *new_mem_obj; - void *ret = NULL; - - NMA_LOCK(); - TAILQ_FOREACH(mem_obj, &nm_mem->nm_molist, nmo_next) { - if (mem_obj->nmo_used != 0 || mem_obj->nmo_size < size) - continue; - - new_mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, - M_WAITOK | M_ZERO); - TAILQ_INSERT_BEFORE(mem_obj, new_mem_obj, nmo_next); - - new_mem_obj->nmo_used = 1; - new_mem_obj->nmo_size = size; - new_mem_obj->nmo_data = mem_obj->nmo_data; - memset(new_mem_obj->nmo_data, 0, new_mem_obj->nmo_size); - - mem_obj->nmo_size -= size; - mem_obj->nmo_data = (char *) mem_obj->nmo_data + size; - if (mem_obj->nmo_size == 0) { - TAILQ_REMOVE(&nm_mem->nm_molist, mem_obj, - nmo_next); - free(mem_obj, M_NETMAP); - } - - ret = new_mem_obj->nmo_data; - - break; - } - NMA_UNLOCK(); - ND("%s: %d bytes at %p", msg, size, ret); - - return (ret); -} - -/* - * Return the memory to the allocator. - * - * While freeing a memory object, we try to merge adjacent chunks in - * order to reduce memory fragmentation. - */ -static void -netmap_free(void *addr, const char *msg) -{ - size_t size; - struct netmap_mem_obj *cur, *prev, *next; - - if (addr == NULL) { - D("NULL addr for %s", msg); - return; - } - - NMA_LOCK(); - TAILQ_FOREACH(cur, &nm_mem->nm_molist, nmo_next) { - if (cur->nmo_data == addr && cur->nmo_used) - break; - } - if (cur == NULL) { - NMA_UNLOCK(); - D("invalid addr %s %p", msg, addr); - return; - } - - size = cur->nmo_size; - cur->nmo_used = 0; - - /* merge current chunk of memory with the previous one, - if present. */ - prev = TAILQ_PREV(cur, netmap_mem_obj_h, nmo_next); - if (prev && prev->nmo_used == 0) { - TAILQ_REMOVE(&nm_mem->nm_molist, cur, nmo_next); - prev->nmo_size += cur->nmo_size; - free(cur, M_NETMAP); - cur = prev; - } - - /* merge with the next one */ - next = TAILQ_NEXT(cur, nmo_next); - if (next && next->nmo_used == 0) { - TAILQ_REMOVE(&nm_mem->nm_molist, next, nmo_next); - cur->nmo_size += next->nmo_size; - free(next, M_NETMAP); - } - NMA_UNLOCK(); - ND("freed %s %d bytes at %p", msg, size, addr); -} - - -/* - * Create and return a new ``netmap_if`` object, and possibly also - * rings and packet buffors. - * - * Return NULL on failure. - */ -static void * -netmap_if_new(const char *ifname, struct netmap_adapter *na) -{ - struct netmap_if *nifp; - struct netmap_ring *ring; - struct netmap_kring *kring; - char *buff; - u_int i, len, ofs, numdesc; - u_int nrx = na->num_rx_queues + 1; /* shorthand, include stack queue */ - u_int ntx = na->num_tx_queues + 1; /* shorthand, include stack queue */ - - /* - * the descriptor is followed inline by an array of offsets - * to the tx and rx rings in the shared memory region. - */ - len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t); - nifp = netmap_if_malloc(len); - if (nifp == NULL) - return (NULL); - - /* initialize base fields */ - *(int *)(uintptr_t)&nifp->ni_rx_queues = na->num_rx_queues; - *(int *)(uintptr_t)&nifp->ni_tx_queues = na->num_tx_queues; - strncpy(nifp->ni_name, ifname, IFNAMSIZ); - - (na->refcount)++; /* XXX atomic ? we are under lock */ - if (na->refcount > 1) - goto final; - - /* - * First instance. Allocate the netmap rings - * (one for each hw queue, one pair for the host). - * The rings are contiguous, but have variable size. - * The entire block is reachable at - * na->tx_rings[0] - */ - len = (ntx + nrx) * sizeof(struct netmap_ring) + - (ntx * na->num_tx_desc + nrx * na->num_rx_desc) * - sizeof(struct netmap_slot); - buff = netmap_ring_malloc(len); - if (buff == NULL) { - D("failed to allocate %d bytes for %s shadow ring", - len, ifname); -error: - (na->refcount)--; - netmap_if_free(nifp); - return (NULL); - } - /* Check whether we have enough buffers */ - len = ntx * na->num_tx_desc + nrx * na->num_rx_desc; - NMA_LOCK(); - if (nm_buf_pool.free < len) { - NMA_UNLOCK(); - netmap_free(buff, "not enough bufs"); - goto error; - } - /* - * in the kring, store the pointers to the shared rings - * and initialize the rings. We are under NMA_LOCK(). - */ - ofs = 0; - for (i = 0; i < ntx; i++) { /* Transmit rings */ - kring = &na->tx_rings[i]; - numdesc = na->num_tx_desc; - bzero(kring, sizeof(*kring)); - kring->na = na; - - ring = kring->ring = (struct netmap_ring *)(buff + ofs); - *(ssize_t *)(uintptr_t)&ring->buf_ofs = - nm_buf_pool.base - (char *)ring; - ND("txring[%d] at %p ofs %d", i, ring, ring->buf_ofs); - *(uint32_t *)(uintptr_t)&ring->num_slots = - kring->nkr_num_slots = numdesc; - - /* - * IMPORTANT: - * Always keep one slot empty, so we can detect new - * transmissions comparing cur and nr_hwcur (they are - * the same only if there are no new transmissions). - */ - ring->avail = kring->nr_hwavail = numdesc - 1; - ring->cur = kring->nr_hwcur = 0; - *(uint16_t *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE; - netmap_new_bufs(nifp, ring->slot, numdesc); - - ofs += sizeof(struct netmap_ring) + - numdesc * sizeof(struct netmap_slot); - } - - for (i = 0; i < nrx; i++) { /* Receive rings */ - kring = &na->rx_rings[i]; - numdesc = na->num_rx_desc; - bzero(kring, sizeof(*kring)); - kring->na = na; - - ring = kring->ring = (struct netmap_ring *)(buff + ofs); - *(ssize_t *)(uintptr_t)&ring->buf_ofs = - nm_buf_pool.base - (char *)ring; - ND("rxring[%d] at %p offset %d", i, ring, ring->buf_ofs); - *(uint32_t *)(uintptr_t)&ring->num_slots = - kring->nkr_num_slots = numdesc; - ring->cur = kring->nr_hwcur = 0; - ring->avail = kring->nr_hwavail = 0; /* empty */ - *(uint16_t *)(uintptr_t)&ring->nr_buf_size = NETMAP_BUF_SIZE; - netmap_new_bufs(nifp, ring->slot, numdesc); - ofs += sizeof(struct netmap_ring) + - numdesc * sizeof(struct netmap_slot); - } - NMA_UNLOCK(); - // XXX initialize the selrecord structs. - -final: - /* - * fill the slots for the rx and tx queues. They contain the offset - * between the ring and nifp, so the information is usable in - * userspace to reach the ring from the nifp. - */ - for (i = 0; i < ntx; i++) { - *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i] = - (char *)na->tx_rings[i].ring - (char *)nifp; - } - for (i = 0; i < nrx; i++) { - *(ssize_t *)(uintptr_t)&nifp->ring_ofs[i+ntx] = - (char *)na->rx_rings[i].ring - (char *)nifp; - } - return (nifp); -} - -/* - * Initialize the memory allocator. - * - * Create the descriptor for the memory , allocate the pool of memory - * and initialize the list of memory objects with a single chunk - * containing the whole pre-allocated memory marked as free. - * - * Start with a large size, then halve as needed if we fail to - * allocate the block. While halving, always add one extra page - * because buffers 0 and 1 are used for special purposes. - * Return 0 on success, errno otherwise. - */ -static int -netmap_memory_init(void) -{ - struct netmap_mem_obj *mem_obj; - void *buf = NULL; - int i, n, sz = NETMAP_MEMORY_SIZE; - int extra_sz = 0; // space for rings and two spare buffers - - for (; sz >= 1<<20; sz >>=1) { - extra_sz = sz/200; - extra_sz = (extra_sz + 2*PAGE_SIZE - 1) & ~(PAGE_SIZE-1); - buf = contigmalloc(sz + extra_sz, - M_NETMAP, - M_WAITOK | M_ZERO, - 0, /* low address */ - -1UL, /* high address */ - PAGE_SIZE, /* alignment */ - 0 /* boundary */ - ); - if (buf) - break; - } - if (buf == NULL) - return (ENOMEM); - sz += extra_sz; - nm_mem = malloc(sizeof(struct netmap_mem_d), M_NETMAP, - M_WAITOK | M_ZERO); - mtx_init(&nm_mem->nm_mtx, "netmap memory allocator lock", NULL, - MTX_DEF); - TAILQ_INIT(&nm_mem->nm_molist); - nm_mem->nm_buffer = buf; - nm_mem->nm_totalsize = sz; - - /* - * A buffer takes 2k, a slot takes 8 bytes + ring overhead, - * so the ratio is 200:1. In other words, we can use 1/200 of - * the memory for the rings, and the rest for the buffers, - * and be sure we never run out. - */ - nm_mem->nm_size = sz/200; - nm_mem->nm_buf_start = - (nm_mem->nm_size + PAGE_SIZE - 1) & ~(PAGE_SIZE-1); - nm_mem->nm_buf_len = sz - nm_mem->nm_buf_start; - - nm_buf_pool.base = nm_mem->nm_buffer; - nm_buf_pool.base += nm_mem->nm_buf_start; - netmap_buffer_base = nm_buf_pool.base; - D("netmap_buffer_base %p (offset %d)", - netmap_buffer_base, (int)nm_mem->nm_buf_start); - /* number of buffers, they all start as free */ - - netmap_total_buffers = nm_buf_pool.total_buffers = - nm_mem->nm_buf_len / NETMAP_BUF_SIZE; - nm_buf_pool.bufsize = NETMAP_BUF_SIZE; - - D("Have %d MB, use %dKB for rings, %d buffers at %p", - (sz >> 20), (int)(nm_mem->nm_size >> 10), - nm_buf_pool.total_buffers, nm_buf_pool.base); - - /* allocate and initialize the bitmap. Entry 0 is considered - * always busy (used as default when there are no buffers left). - */ - n = (nm_buf_pool.total_buffers + 31) / 32; - nm_buf_pool.bitmap = malloc(sizeof(uint32_t) * n, M_NETMAP, - M_WAITOK | M_ZERO); - nm_buf_pool.bitmap[0] = ~3; /* slot 0 and 1 always busy */ - for (i = 1; i < n; i++) - nm_buf_pool.bitmap[i] = ~0; - nm_buf_pool.free = nm_buf_pool.total_buffers - 2; - - mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, - M_WAITOK | M_ZERO); - TAILQ_INSERT_HEAD(&nm_mem->nm_molist, mem_obj, nmo_next); - mem_obj->nmo_used = 0; - mem_obj->nmo_size = nm_mem->nm_size; - mem_obj->nmo_data = nm_mem->nm_buffer; - - return (0); -} - - -/* - * Finalize the memory allocator. - * - * Free all the memory objects contained inside the list, and deallocate - * the pool of memory; finally free the memory allocator descriptor. - */ -static void -netmap_memory_fini(void) -{ - struct netmap_mem_obj *mem_obj; - - while (!TAILQ_EMPTY(&nm_mem->nm_molist)) { - mem_obj = TAILQ_FIRST(&nm_mem->nm_molist); - TAILQ_REMOVE(&nm_mem->nm_molist, mem_obj, nmo_next); - if (mem_obj->nmo_used == 1) { - printf("netmap: leaked %d bytes at %p\n", - (int)mem_obj->nmo_size, - mem_obj->nmo_data); - } - free(mem_obj, M_NETMAP); - } - contigfree(nm_mem->nm_buffer, nm_mem->nm_totalsize, M_NETMAP); - // XXX mutex_destroy(nm_mtx); - free(nm_mem, M_NETMAP); -} -/*------------- end of memory allocator -----------------*/ - +/*------------- memory allocator -----------------*/ +#ifdef NETMAP_MEM2 +#include "netmap_mem2.c" +#else /* !NETMAP_MEM2 */ +#include "netmap_mem1.c" +#endif /* !NETMAP_MEM2 */ +/*------------ end of memory allocator ----------*/ /* Structure associated to each thread which registered an interface. */ struct netmap_priv_d { @@ -837,7 +343,7 @@ netmap_sync_from_host(struct netmap_adap if (j != k) { n = k >= j ? k - j : k + lim - j; kring->nr_hwavail -= n; - kring->nr_hwcur = k; + kring->nr_hwcur = k; } k = ring->avail = kring->nr_hwavail - resvd; if (k == 0 && td) @@ -1150,27 +656,28 @@ error: /* find the last ring to scan */ lim = priv->np_qlast; if (lim == NETMAP_HW_RING) - lim = (cmd == NIOCTXSYNC) ? na->num_tx_queues : na->num_rx_queues; + lim = (cmd == NIOCTXSYNC) ? + na->num_tx_queues : na->num_rx_queues; for (i = priv->np_qfirst; i < lim; i++) { - if (cmd == NIOCTXSYNC) { - struct netmap_kring *kring = &na->tx_rings[i]; - if (netmap_verbose & NM_VERB_TXSYNC) - D("sync tx ring %d cur %d hwcur %d", - i, kring->ring->cur, - kring->nr_hwcur); - na->nm_txsync(ifp, i, 1 /* do lock */); - if (netmap_verbose & NM_VERB_TXSYNC) - D("after sync tx ring %d cur %d hwcur %d", - i, kring->ring->cur, - kring->nr_hwcur); - } else { - na->nm_rxsync(ifp, i, 1 /* do lock */); - microtime(&na->rx_rings[i].ring->ts); - } + if (cmd == NIOCTXSYNC) { + struct netmap_kring *kring = &na->tx_rings[i]; + if (netmap_verbose & NM_VERB_TXSYNC) + D("pre txsync ring %d cur %d hwcur %d", + i, kring->ring->cur, + kring->nr_hwcur); + na->nm_txsync(ifp, i, 1 /* do lock */); + if (netmap_verbose & NM_VERB_TXSYNC) + D("post txsync ring %d cur %d hwcur %d", + i, kring->ring->cur, + kring->nr_hwcur); + } else { + na->nm_rxsync(ifp, i, 1 /* do lock */); + microtime(&na->rx_rings[i].ring->ts); + } } - break; + break; case BIOCIMMEDIATE: case BIOCGHDRCMPLT: @@ -1645,13 +1152,13 @@ netmap_reset(struct netmap_adapter *na, * Default functions to handle rx/tx interrupts * we have 4 cases: * 1 ring, single lock: - * lock(core); wake(i=0); unlock(core) + * lock(core); wake(i=0); unlock(core) * N rings, single lock: - * lock(core); wake(i); wake(N+1) unlock(core) + * lock(core); wake(i); wake(N+1) unlock(core) * 1 ring, separate locks: (i=0) - * lock(i); wake(i); unlock(i) + * lock(i); wake(i); unlock(i) * N rings, separate locks: - * lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core) + * lock(i); wake(i); unlock(i); lock(core) wake(N+1) unlock(core) * work_done is non-null on the RX path. */ int @@ -1689,7 +1196,7 @@ netmap_rx_irq(struct ifnet *ifp, int q, selwakeuppri(main_wq, PI_NET); mtx_unlock(&na->core_lock); } - *work_done = 1; /* do not fire napi again */ + *work_done = 1; /* do not fire napi again */ return 1; } Modified: head/sys/dev/netmap/netmap_kern.h ============================================================================== --- head/sys/dev/netmap/netmap_kern.h Thu Apr 12 11:23:24 2012 (r234173) +++ head/sys/dev/netmap/netmap_kern.h Thu Apr 12 11:27:09 2012 (r234174) @@ -86,7 +86,7 @@ struct netmap_kring { u_int nr_hwcur; int nr_hwavail; u_int nr_kflags; /* private driver flags */ -#define NKR_PENDINTR 0x1 // Pending interrupt. +#define NKR_PENDINTR 0x1 // Pending interrupt. u_int nkr_num_slots; int nkr_hwofs; /* offset between NIC and netmap ring */ @@ -208,7 +208,7 @@ struct netmap_slot *netmap_reset(struct int netmap_ring_reinit(struct netmap_kring *); extern int netmap_buf_size; -#define NETMAP_BUF_SIZE netmap_buf_size +#define NETMAP_BUF_SIZE netmap_buf_size extern int netmap_mitigate; extern int netmap_no_pendintr; extern u_int netmap_total_buffers; @@ -303,7 +303,7 @@ NMB(struct netmap_slot *slot) { uint32_t i = slot->buf_idx; return (i >= netmap_total_buffers) ? netmap_buffer_base : - netmap_buffer_base + (i *NETMAP_BUF_SIZE); + netmap_buffer_base + (i * NETMAP_BUF_SIZE); } static inline void * @@ -311,7 +311,7 @@ PNMB(struct netmap_slot *slot, uint64_t { uint32_t i = slot->buf_idx; void *ret = (i >= netmap_total_buffers) ? netmap_buffer_base : - netmap_buffer_base + (i *NETMAP_BUF_SIZE); + netmap_buffer_base + (i * NETMAP_BUF_SIZE); *pp = vtophys(ret); return ret; } @@ -319,8 +319,4 @@ PNMB(struct netmap_slot *slot, uint64_t /* default functions to handle rx/tx interrupts */ int netmap_rx_irq(struct ifnet *, int, int *); #define netmap_tx_irq(_n, _q) netmap_rx_irq(_n, _q, NULL) -#ifdef __linux__ -#define bus_dmamap_sync(_a, _b, _c) // wmb() or rmb() ? -netdev_tx_t netmap_start_linux(struct sk_buff *skb, struct net_device *dev); -#endif #endif /* _NET_NETMAP_KERN_H_ */ Added: head/sys/dev/netmap/netmap_mem1.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/dev/netmap/netmap_mem1.c Thu Apr 12 11:27:09 2012 (r234174) @@ -0,0 +1,521 @@ +/* + * Copyright (C) 2011 Matteo Landi, Luigi Rizzo. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * $FreeBSD$ + * + * The original netmap memory allocator, using a single large + * chunk of memory allocated with contigmalloc. + */ + +/* + * Default amount of memory pre-allocated by the module. + * We start with a large size and then shrink our demand + * according to what is avalable when the module is loaded. + */ +#define NETMAP_MEMORY_SIZE (64 * 1024 * PAGE_SIZE) +static void * netmap_malloc(size_t size, const char *msg); +static void netmap_free(void *addr, const char *msg); + +#define netmap_if_malloc(len) netmap_malloc(len, "nifp") +#define netmap_if_free(v) netmap_free((v), "nifp") + +#define netmap_ring_malloc(len) netmap_malloc(len, "ring") +#define netmap_free_rings(na) \ + netmap_free((na)->tx_rings[0].ring, "shadow rings"); + +/* + * Allocator for a pool of packet buffers. For each buffer we have + * one entry in the bitmap to signal the state. Allocation scans + * the bitmap, but since this is done only on attach, we are not + * too worried about performance + * XXX if we need to allocate small blocks, a translation + * table is used both for kernel virtual address and physical + * addresses. + */ +struct netmap_buf_pool { + u_int total_buffers; /* total buffers. */ + u_int free; + u_int bufsize; + char *base; /* buffer base address */ + uint32_t *bitmap; /* one bit per buffer, 1 means free */ +}; +struct netmap_buf_pool nm_buf_pool; +SYSCTL_INT(_dev_netmap, OID_AUTO, total_buffers, + CTLFLAG_RD, &nm_buf_pool.total_buffers, 0, "total_buffers"); +SYSCTL_INT(_dev_netmap, OID_AUTO, free_buffers, + CTLFLAG_RD, &nm_buf_pool.free, 0, "free_buffers"); + + +/* + * Allocate n buffers from the ring, and fill the slot. + * Buffer 0 is the 'junk' buffer. + */ +static void +netmap_new_bufs(struct netmap_if *nifp __unused, + struct netmap_slot *slot, u_int n) +{ + struct netmap_buf_pool *p = &nm_buf_pool; + uint32_t bi = 0; /* index in the bitmap */ + uint32_t mask, j, i = 0; /* slot counter */ + + if (n > p->free) { + D("only %d out of %d buffers available", i, n); + return; + } + /* termination is guaranteed by p->free */ + while (i < n && p->free > 0) { + uint32_t cur = p->bitmap[bi]; + if (cur == 0) { /* bitmask is fully used */ + bi++; + continue; + } + /* locate a slot */ + for (j = 0, mask = 1; (cur & mask) == 0; j++, mask <<= 1) ; + p->bitmap[bi] &= ~mask; /* slot in use */ + p->free--; + slot[i].buf_idx = bi*32+j; + slot[i].len = p->bufsize; + slot[i].flags = NS_BUF_CHANGED; + i++; + } + ND("allocated %d buffers, %d available", n, p->free); +} + + +static void +netmap_free_buf(struct netmap_if *nifp __unused, uint32_t i) +{ + struct netmap_buf_pool *p = &nm_buf_pool; + + uint32_t pos, mask; + if (i >= p->total_buffers) { + D("invalid free index %d", i); + return; + } + pos = i / 32; + mask = 1 << (i % 32); + if (p->bitmap[pos] & mask) { + D("slot %d already free", i); + return; + } + p->bitmap[pos] |= mask; + p->free++; +} + + +/* Descriptor of the memory objects handled by our memory allocator. */ +struct netmap_mem_obj { + TAILQ_ENTRY(netmap_mem_obj) nmo_next; /* next object in the + chain. */ + int nmo_used; /* flag set on used memory objects. */ + size_t nmo_size; /* size of the memory area reserved for the + object. */ + void *nmo_data; /* pointer to the memory area. */ +}; + +/* Wrap our memory objects to make them ``chainable``. */ +TAILQ_HEAD(netmap_mem_obj_h, netmap_mem_obj); + + +/* Descriptor of our custom memory allocator. */ +struct netmap_mem_d { + struct mtx nm_mtx; /* lock used to handle the chain of memory + objects. */ + struct netmap_mem_obj_h nm_molist; /* list of memory objects */ + size_t nm_size; /* total amount of memory used for rings etc. */ + size_t nm_totalsize; /* total amount of allocated memory + (the difference is used for buffers) */ + size_t nm_buf_start; /* offset of packet buffers. + This is page-aligned. */ + size_t nm_buf_len; /* total memory for buffers */ + void *nm_buffer; /* pointer to the whole pre-allocated memory + area. */ +}; + +/* Shorthand to compute a netmap interface offset. */ +#define netmap_if_offset(v) \ + ((char *) (v) - (char *) nm_mem->nm_buffer) +/* .. and get a physical address given a memory offset */ +#define netmap_ofstophys(o) \ + (vtophys(nm_mem->nm_buffer) + (o)) + + +/*------ netmap memory allocator -------*/ +/* + * Request for a chunk of memory. + * + * Memory objects are arranged into a list, hence we need to walk this + * list until we find an object with the needed amount of data free. + * This sounds like a completely inefficient implementation, but given + * the fact that data allocation is done once, we can handle it + * flawlessly. + * + * Return NULL on failure. + */ +static void * +netmap_malloc(size_t size, __unused const char *msg) +{ + struct netmap_mem_obj *mem_obj, *new_mem_obj; + void *ret = NULL; + + NMA_LOCK(); + TAILQ_FOREACH(mem_obj, &nm_mem->nm_molist, nmo_next) { + if (mem_obj->nmo_used != 0 || mem_obj->nmo_size < size) + continue; + + new_mem_obj = malloc(sizeof(struct netmap_mem_obj), M_NETMAP, + M_WAITOK | M_ZERO); + TAILQ_INSERT_BEFORE(mem_obj, new_mem_obj, nmo_next); + + new_mem_obj->nmo_used = 1; + new_mem_obj->nmo_size = size; + new_mem_obj->nmo_data = mem_obj->nmo_data; + memset(new_mem_obj->nmo_data, 0, new_mem_obj->nmo_size); + + mem_obj->nmo_size -= size; + mem_obj->nmo_data = (char *) mem_obj->nmo_data + size; + if (mem_obj->nmo_size == 0) { + TAILQ_REMOVE(&nm_mem->nm_molist, mem_obj, + nmo_next); + free(mem_obj, M_NETMAP); + } + + ret = new_mem_obj->nmo_data; + + break; + } + NMA_UNLOCK(); + ND("%s: %d bytes at %p", msg, size, ret); + + return (ret); +} + +/* + * Return the memory to the allocator. + * + * While freeing a memory object, we try to merge adjacent chunks in + * order to reduce memory fragmentation. + */ +static void +netmap_free(void *addr, const char *msg) +{ + size_t size; + struct netmap_mem_obj *cur, *prev, *next; + + if (addr == NULL) { + D("NULL addr for %s", msg); + return; + } + + NMA_LOCK(); + TAILQ_FOREACH(cur, &nm_mem->nm_molist, nmo_next) { + if (cur->nmo_data == addr && cur->nmo_used) + break; + } + if (cur == NULL) { + NMA_UNLOCK(); + D("invalid addr %s %p", msg, addr); + return; + } + + size = cur->nmo_size; + cur->nmo_used = 0; + + /* merge current chunk of memory with the previous one, + if present. */ + prev = TAILQ_PREV(cur, netmap_mem_obj_h, nmo_next); + if (prev && prev->nmo_used == 0) { + TAILQ_REMOVE(&nm_mem->nm_molist, cur, nmo_next); + prev->nmo_size += cur->nmo_size; + free(cur, M_NETMAP); + cur = prev; + } + + /* merge with the next one */ + next = TAILQ_NEXT(cur, nmo_next); + if (next && next->nmo_used == 0) { + TAILQ_REMOVE(&nm_mem->nm_molist, next, nmo_next); + cur->nmo_size += next->nmo_size; + free(next, M_NETMAP); + } + NMA_UNLOCK(); + ND("freed %s %d bytes at %p", msg, size, addr); +} + + +/* + * Create and return a new ``netmap_if`` object, and possibly also + * rings and packet buffors. + * + * Return NULL on failure. + */ +static void * +netmap_if_new(const char *ifname, struct netmap_adapter *na) +{ + struct netmap_if *nifp; + struct netmap_ring *ring; + struct netmap_kring *kring; + char *buff; + u_int i, len, ofs, numdesc; + u_int nrx = na->num_rx_queues + 1; /* shorthand, include stack queue */ + u_int ntx = na->num_tx_queues + 1; /* shorthand, include stack queue */ + + /* + * the descriptor is followed inline by an array of offsets + * to the tx and rx rings in the shared memory region. + */ + len = sizeof(struct netmap_if) + (nrx + ntx) * sizeof(ssize_t); + nifp = netmap_if_malloc(len); + if (nifp == NULL) + return (NULL); + + /* initialize base fields */ + *(int *)(uintptr_t)&nifp->ni_rx_queues = na->num_rx_queues; + *(int *)(uintptr_t)&nifp->ni_tx_queues = na->num_tx_queues; + strncpy(nifp->ni_name, ifname, IFNAMSIZ); + + (na->refcount)++; /* XXX atomic ? we are under lock */ + if (na->refcount > 1) + goto final; + + /* + * First instance. Allocate the netmap rings + * (one for each hw queue, one pair for the host). + * The rings are contiguous, but have variable size. + * The entire block is reachable at + * na->tx_rings[0] + */ + len = (ntx + nrx) * sizeof(struct netmap_ring) + + (ntx * na->num_tx_desc + nrx * na->num_rx_desc) * + sizeof(struct netmap_slot); + buff = netmap_ring_malloc(len); + if (buff == NULL) { + D("failed to allocate %d bytes for %s shadow ring", + len, ifname); +error: + (na->refcount)--; + netmap_if_free(nifp); + return (NULL); + } + /* Check whether we have enough buffers */ + len = ntx * na->num_tx_desc + nrx * na->num_rx_desc; + NMA_LOCK(); + if (nm_buf_pool.free < len) { + NMA_UNLOCK(); + netmap_free(buff, "not enough bufs"); + goto error; + } + /* + * in the kring, store the pointers to the shared rings + * and initialize the rings. We are under NMA_LOCK(). + */ + ofs = 0; + for (i = 0; i < ntx; i++) { /* Transmit rings */ + kring = &na->tx_rings[i]; + numdesc = na->num_tx_desc; + bzero(kring, sizeof(*kring)); + kring->na = na; + + ring = kring->ring = (struct netmap_ring *)(buff + ofs); + *(ssize_t *)(uintptr_t)&ring->buf_ofs = + nm_buf_pool.base - (char *)ring; + ND("txring[%d] at %p ofs %d", i, ring, ring->buf_ofs); + *(uint32_t *)(uintptr_t)&ring->num_slots = + kring->nkr_num_slots = numdesc; + + /* + * IMPORTANT: + * Always keep one slot empty, so we can detect new *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201204121127.q3CBR9VO026345>