Date: Sun, 24 May 2015 01:48:34 +0000 (UTC) From: Patrick Kelsey <pkelsey@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r283343 - stable/10/sys/dev/netmap Message-ID: <201505240148.t4O1mYVM038526@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: pkelsey Date: Sun May 24 01:48:33 2015 New Revision: 283343 URL: https://svnweb.freebsd.org/changeset/base/283343 Log: MFC r282978: When a netmap process terminates without the full set of buffers it was granted via rings and ni_bufs_list_head represented in those rings and lists (e.g., via SIGKILL), those buffers are no longer available for subsequent users for the lifetime of the system. To mitigate this resource leak, reset the allocator state when the last ref to that allocator is released. Note that this only recovers leaked resources for an allocator when there are no longer any users of that allocator, so there remain circumstances in which leaked allocator resources may not ever be recovered - consider a set of multiple netmap processes that are all using the same allocator (say, the global allocator) where members of that set may be killed and restarted over time but at any given point there is one member of that set running. Modified: stable/10/sys/dev/netmap/netmap_mem2.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/dev/netmap/netmap_mem2.c ============================================================================== --- stable/10/sys/dev/netmap/netmap_mem2.c Sun May 24 01:09:51 2015 (r283342) +++ stable/10/sys/dev/netmap/netmap_mem2.c Sun May 24 01:48:33 2015 (r283343) @@ -130,9 +130,9 @@ struct netmap_mem_d { /* the three allocators */ struct netmap_obj_pool pools[NETMAP_POOLS_NR]; - netmap_mem_config_t config; - netmap_mem_finalize_t finalize; - netmap_mem_deref_t deref; + netmap_mem_config_t config; /* called with NMA_LOCK held */ + netmap_mem_finalize_t finalize; /* called with NMA_LOCK held */ + netmap_mem_deref_t deref; /* called with NMA_LOCK held */ nm_memid_t nm_id; /* allocator identifier */ int nm_grp; /* iommu groupd id */ @@ -751,6 +751,12 @@ netmap_reset_obj_allocator(struct netmap u_int i; size_t sz = p->_clustsize; + /* + * Free each cluster allocated in + * netmap_finalize_obj_allocator(). The cluster start + * addresses are stored at multiples of p->_clusterentries + * in the lut. + */ for (i = 0; i < p->objtotal; i += p->_clustentries) { if (p->lut[i].vaddr) contigfree(p->lut[i].vaddr, sz, M_NETMAP); @@ -929,6 +935,7 @@ netmap_finalize_obj_allocator(struct net if (i % p->_clustentries == 0 && p->lut[i].vaddr) contigfree(p->lut[i].vaddr, n, M_NETMAP); + p->lut[i].vaddr = NULL; } out: p->objtotal = i; @@ -936,6 +943,17 @@ netmap_finalize_obj_allocator(struct net p->numclusters = (i + p->_clustentries - 1) / p->_clustentries; break; } + /* + * Set bitmap and lut state for all buffers in the current + * cluster. + * + * [i, lim) is the set of buffer indexes that cover the + * current cluster. + * + * 'clust' is really the address of the current buffer in + * the current cluster as we index through it with a stride + * of p->_objsize. + */ for (; i < lim; i++, clust += p->_objsize) { p->bitmap[ (i>>5) ] |= ( 1 << (i & 31) ); p->lut[i].vaddr = clust; @@ -1092,10 +1110,8 @@ static int netmap_mem_private_finalize(struct netmap_mem_d *nmd) { int err; - NMA_LOCK(nmd); nmd->refcount++; err = netmap_mem_finalize_all(nmd); - NMA_UNLOCK(nmd); return err; } @@ -1103,10 +1119,8 @@ netmap_mem_private_finalize(struct netma static void netmap_mem_private_deref(struct netmap_mem_d *nmd) { - NMA_LOCK(nmd); if (--nmd->refcount <= 0) netmap_mem_reset_all(nmd); - NMA_UNLOCK(nmd); } @@ -1242,10 +1256,7 @@ static int netmap_mem_global_finalize(struct netmap_mem_d *nmd) { int err; - - NMA_LOCK(nmd); - - + /* update configuration if changed */ if (netmap_mem_global_config(nmd)) goto out; @@ -1268,8 +1279,6 @@ out: nmd->refcount--; err = nmd->lasterr; - NMA_UNLOCK(nmd); - return err; } @@ -1518,7 +1527,6 @@ netmap_mem_if_delete(struct netmap_adapt static void netmap_mem_global_deref(struct netmap_mem_d *nmd) { - NMA_LOCK(nmd); nmd->refcount--; if (!nmd->refcount) @@ -1526,7 +1534,6 @@ netmap_mem_global_deref(struct netmap_me if (netmap_verbose) D("refcount = %d", nmd->refcount); - NMA_UNLOCK(nmd); } int @@ -1535,7 +1542,9 @@ netmap_mem_finalize(struct netmap_mem_d if (nm_mem_assign_group(nmd, na->pdev) < 0) { return ENOMEM; } else { + NMA_LOCK(nmd); nmd->finalize(nmd); + NMA_UNLOCK(nmd); } if (!nmd->lasterr && na->pdev) @@ -1549,6 +1558,48 @@ netmap_mem_deref(struct netmap_mem_d *nm { NMA_LOCK(nmd); netmap_mem_unmap(&nmd->pools[NETMAP_BUF_POOL], na); + if (nmd->refcount == 1) { + u_int i; + + /* + * Reset the allocator when it falls out of use so that any + * pool resources leaked by unclean application exits are + * reclaimed. + */ + for (i = 0; i < NETMAP_POOLS_NR; i++) { + struct netmap_obj_pool *p; + u_int j; + + p = &nmd->pools[i]; + p->objfree = p->objtotal; + /* + * Reproduce the net effect of the M_ZERO malloc() + * and marking of free entries in the bitmap that + * occur in finalize_obj_allocator() + */ + memset(p->bitmap, + '\0', + sizeof(uint32_t) * ((p->objtotal + 31) / 32)); + + /* + * Set all the bits in the bitmap that have + * corresponding buffers to 1 to indicate they are + * free. + */ + for (j = 0; j < p->objtotal; j++) { + if (p->lut[j].vaddr != NULL) { + p->bitmap[ (j>>5) ] |= ( 1 << (j & 31) ); + } + } + } + + /* + * Per netmap_mem_finalize_all(), + * buffers 0 and 1 are reserved + */ + nmd->pools[NETMAP_BUF_POOL].objfree -= 2; + nmd->pools[NETMAP_BUF_POOL].bitmap[0] = ~3; + } + nmd->deref(nmd); NMA_UNLOCK(nmd); - return nmd->deref(nmd); }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201505240148.t4O1mYVM038526>