Date: Wed, 28 Jan 2009 16:37:49 +0000 (UTC) From: Alan Cox <alc@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r187829 - in user/alc/zerocopy/sys: kern sys Message-ID: <200901281637.n0SGbnqo026334@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: alc Date: Wed Jan 28 16:37:49 2009 New Revision: 187829 URL: http://svn.freebsd.org/changeset/base/187829 Log: Adapt the zero-copy transmit mechanism to work with a new explicit API. At the moment, the API is a straw man, and not a serious or complete proposal. However, it does allow for testing the implementation. Modified: user/alc/zerocopy/sys/kern/uipc_cow.c user/alc/zerocopy/sys/kern/uipc_socket.c user/alc/zerocopy/sys/kern/uipc_syscalls.c user/alc/zerocopy/sys/sys/socket.h user/alc/zerocopy/sys/sys/socketvar.h Modified: user/alc/zerocopy/sys/kern/uipc_cow.c ============================================================================== --- user/alc/zerocopy/sys/kern/uipc_cow.c Wed Jan 28 16:23:21 2009 (r187828) +++ user/alc/zerocopy/sys/kern/uipc_cow.c Wed Jan 28 16:37:49 2009 (r187829) @@ -63,39 +63,12 @@ struct netsend_cow_stats { int fail_not_mapped; int fail_sf_buf; int success; - int iodone; }; static struct netsend_cow_stats socow_stats; -static void socow_iodone(void *addr, void *args); - -static void -socow_iodone(void *addr, void *args) -{ - struct sf_buf *sf; - vm_page_t pp; - - sf = args; - pp = sf_buf_page(sf); - sf_buf_free(sf); - /* remove COW mapping */ - vm_page_lock_queues(); - vm_page_cowclear(pp); - vm_page_unwire(pp, 0); - /* - * Check for the object going away on us. This can - * happen since we don't hold a reference to it. - * If so, we're responsible for freeing the page. - */ - if (pp->wire_count == 0 && pp->object == NULL) - vm_page_free(pp); - vm_page_unlock_queues(); - socow_stats.iodone++; -} - int -socow_setup(struct mbuf *m0, struct uio *uio) +socow_setup(struct mbuf *m0, struct uio *uio, struct sendfile_sync *sfs) { struct sf_buf *sf; vm_page_t pp; @@ -125,19 +98,10 @@ socow_setup(struct mbuf *m0, struct uio return(0); } - /* - * set up COW - */ - vm_page_lock_queues(); - if (vm_page_cowsetup(pp) != 0) { - vm_page_unhold(pp); - vm_page_unlock_queues(); - return (0); - } - /* * wire the page for I/O */ + vm_page_lock_queues(); vm_page_wire(pp); vm_page_unhold(pp); vm_page_unlock_queues(); @@ -148,7 +112,6 @@ socow_setup(struct mbuf *m0, struct uio sf = sf_buf_alloc(pp, SFB_CATCH); if (!sf) { vm_page_lock_queues(); - vm_page_cowclear(pp); vm_page_unwire(pp, 0); /* * Check for the object going away on us. This can @@ -164,12 +127,17 @@ socow_setup(struct mbuf *m0, struct uio /* * attach to mbuf */ - MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, socow_iodone, - (void*)sf_buf_kva(sf), sf, M_RDONLY, EXT_SFBUF); + MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, sf_buf_mext, + sfs, sf, M_RDONLY, EXT_SFBUF); m0->m_len = PAGE_SIZE - offset; m0->m_data = (caddr_t)sf_buf_kva(sf) + offset; socow_stats.success++; + if (sfs != NULL) { + mtx_lock(&sfs->mtx); + sfs->count++; + mtx_unlock(&sfs->mtx); + } iov = uio->uio_iov; iov->iov_base = (char *)iov->iov_base + m0->m_len; iov->iov_len -= m0->m_len; Modified: user/alc/zerocopy/sys/kern/uipc_socket.c ============================================================================== --- user/alc/zerocopy/sys/kern/uipc_socket.c Wed Jan 28 16:23:21 2009 (r187828) +++ user/alc/zerocopy/sys/kern/uipc_socket.c Wed Jan 28 16:37:49 2009 (r187829) @@ -105,6 +105,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> +#include <sys/condvar.h> #include <sys/fcntl.h> #include <sys/limits.h> #include <sys/lock.h> @@ -176,14 +177,11 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, numopens &numopensockets, 0, "Number of open sockets"); #ifdef ZERO_COPY_SOCKETS /* These aren't static because they're used in other files. */ -int so_zero_copy_send = 1; int so_zero_copy_receive = 1; SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0, "Zero copy controls"); SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW, &so_zero_copy_receive, 0, "Enable zero copy receive"); -SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW, - &so_zero_copy_send, 0, "Enable zero copy send"); #endif /* ZERO_COPY_SOCKETS */ /* @@ -831,7 +829,7 @@ struct so_zerocopy_stats so_zerocp_stats */ static int sosend_copyin(struct uio *uio, struct mbuf **retmp, int atomic, long *space, - int flags) + int flags, struct sendfile_sync *sfs) { struct mbuf *m, **mp, *top; long len, resid; @@ -857,13 +855,13 @@ sosend_copyin(struct uio *uio, struct mb m->m_pkthdr.rcvif = NULL; } else m = m_get(M_WAITOK, MT_DATA); - if (so_zero_copy_send && + if ((flags & (MSG_AZCS | MSG_SZCS)) != 0 && resid>=PAGE_SIZE && *space>=PAGE_SIZE && uio->uio_iov->iov_len>=PAGE_SIZE) { so_zerocp_stats.size_ok++; so_zerocp_stats.align_ok++; - cow_send = socow_setup(m, uio); + cow_send = socow_setup(m, uio, sfs); len = cow_send; } if (!cow_send) { @@ -934,6 +932,7 @@ int sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, struct mbuf *top, struct mbuf *control, int flags, struct thread *td) { + struct sendfile_sync *sfs = NULL; long space, resid; int clen = 0, error, dontroute; #ifdef ZERO_COPY_SOCKETS @@ -963,6 +962,11 @@ sosend_dgram(struct socket *so, struct s goto out; } + if (flags & MSG_SZCS) { + sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO); + mtx_init(&sfs->mtx, "sendfile", MTX_DEF, 0); + cv_init(&sfs->cv, "sendfile"); + } dontroute = (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; if (td != NULL) @@ -1025,7 +1029,7 @@ sosend_dgram(struct socket *so, struct s top->m_flags |= M_EOR; } else { #ifdef ZERO_COPY_SOCKETS - error = sosend_copyin(uio, &top, atomic, &space, flags); + error = sosend_copyin(uio, &top, atomic, &space, flags, sfs); if (error) goto out; #else @@ -1088,6 +1092,15 @@ out: m_freem(top); if (control != NULL) m_freem(control); + if (sfs != NULL) { + mtx_lock(&sfs->mtx); + if (sfs->count != 0) + cv_wait(&sfs->cv, &sfs->mtx); + KASSERT(sfs->count == 0, ("sendfile sync still busy")); + cv_destroy(&sfs->cv); + mtx_destroy(&sfs->mtx); + free(sfs, M_TEMP); + } return (error); } @@ -1108,6 +1121,7 @@ int sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, struct mbuf *top, struct mbuf *control, int flags, struct thread *td) { + struct sendfile_sync *sfs = NULL; long space, resid; int clen = 0, error, dontroute; int atomic = sosendallatonce(so) || top; @@ -1131,6 +1145,11 @@ sosend_generic(struct socket *so, struct goto out; } + if (flags & MSG_SZCS) { + sfs = malloc(sizeof(*sfs), M_TEMP, M_WAITOK | M_ZERO); + mtx_init(&sfs->mtx, "sendfile", MTX_DEF, 0); + cv_init(&sfs->cv, "sendfile"); + } dontroute = (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && (so->so_proto->pr_flags & PR_ATOMIC); @@ -1213,7 +1232,7 @@ restart: } else { #ifdef ZERO_COPY_SOCKETS error = sosend_copyin(uio, &top, atomic, - &space, flags); + &space, flags, sfs); if (error != 0) goto release; #else @@ -1283,6 +1302,15 @@ out: m_freem(top); if (control != NULL) m_freem(control); + if (sfs != NULL) { + mtx_lock(&sfs->mtx); + if (sfs->count != 0) + cv_wait(&sfs->cv, &sfs->mtx); + KASSERT(sfs->count == 0, ("sendfile sync still busy")); + cv_destroy(&sfs->cv); + mtx_destroy(&sfs->mtx); + free(sfs, M_TEMP); + } return (error); } Modified: user/alc/zerocopy/sys/kern/uipc_syscalls.c ============================================================================== --- user/alc/zerocopy/sys/kern/uipc_syscalls.c Wed Jan 28 16:23:21 2009 (r187828) +++ user/alc/zerocopy/sys/kern/uipc_syscalls.c Wed Jan 28 16:37:49 2009 (r187829) @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> +#include <sys/condvar.h> #include <sys/kernel.h> #include <sys/lock.h> #include <sys/mutex.h> @@ -1676,14 +1677,6 @@ getsockaddr(namp, uaddr, len) return (error); } -#include <sys/condvar.h> - -struct sendfile_sync { - struct mtx mtx; - struct cv cv; - unsigned count; -}; - /* * Detach mapped page and release resources back to the system. */ @@ -1867,8 +1860,7 @@ kern_sendfile(struct thread *td, struct mnw = 1; if (uap->flags & SF_SYNC) { - sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK); - memset(sfs, 0, sizeof *sfs); + sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO); mtx_init(&sfs->mtx, "sendfile", MTX_DEF, 0); cv_init(&sfs->cv, "sendfile"); } Modified: user/alc/zerocopy/sys/sys/socket.h ============================================================================== --- user/alc/zerocopy/sys/sys/socket.h Wed Jan 28 16:23:21 2009 (r187828) +++ user/alc/zerocopy/sys/sys/socket.h Wed Jan 28 16:37:49 2009 (r187829) @@ -465,6 +465,8 @@ struct msghdr { #endif #if __BSD_VISIBLE #define MSG_NOSIGNAL 0x20000 /* do not generate SIGPIPE on EOF */ +#define MSG_AZCS 0x100000 /* asynchronous zero-copy send */ +#define MSG_SZCS 0x200000 /* synchronous zero-copy send */ #endif /* Modified: user/alc/zerocopy/sys/sys/socketvar.h ============================================================================== --- user/alc/zerocopy/sys/sys/socketvar.h Wed Jan 28 16:23:21 2009 (r187828) +++ user/alc/zerocopy/sys/sys/socketvar.h Wed Jan 28 16:37:49 2009 (r187829) @@ -39,6 +39,7 @@ #include <sys/_lock.h> #include <sys/_mutex.h> #include <sys/_sx.h> +#include <sys/condvar.h> #include <sys/sockbuf.h> #include <sys/sockstate.h> #ifdef _KERNEL @@ -286,6 +287,12 @@ struct accept_filter { SLIST_ENTRY(accept_filter) accf_next; }; +struct sendfile_sync { + struct mtx mtx; + struct cv cv; + unsigned count; +}; + #ifdef MALLOC_DECLARE MALLOC_DECLARE(M_ACCF); MALLOC_DECLARE(M_PCB); @@ -314,7 +321,8 @@ int sobind(struct socket *so, struct soc int soclose(struct socket *so); int soconnect(struct socket *so, struct sockaddr *nam, struct thread *td); int soconnect2(struct socket *so1, struct socket *so2); -int socow_setup(struct mbuf *m0, struct uio *uio); +int socow_setup(struct mbuf *m0, struct uio *uio, + struct sendfile_sync *sfs); int socreate(int dom, struct socket **aso, int type, int proto, struct ucred *cred, struct thread *td); int sodisconnect(struct socket *so);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200901281637.n0SGbnqo026334>