Date: Wed, 12 Feb 2014 19:51:13 +0000 (UTC) From: Gleb Smirnoff <glebius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r261807 - in projects/sendfile/sys: kern sys ufs/ffs vm Message-ID: <201402121951.s1CJpDAc060029@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: glebius Date: Wed Feb 12 19:51:12 2014 New Revision: 261807 URL: http://svnweb.freebsd.org/changeset/base/261807 Log: Add new VOP vop_getpages_async(), that is like vop_getpages() but doesn't block on I/O, just returns. The callback function supplied will be executed with the argument supplied after I/O is done and pages are populated and valid. Implement the VOP for the generic vnode pager, tearing the vnode_pager_generic_getpages() into two parts. Sponsored by: Netflix Sponsored by: Nginx, Inc. Modified: projects/sendfile/sys/kern/vfs_default.c projects/sendfile/sys/kern/vnode_if.src projects/sendfile/sys/sys/vnode.h projects/sendfile/sys/ufs/ffs/ffs_vnops.c projects/sendfile/sys/vm/vm_pager.h projects/sendfile/sys/vm/vnode_pager.c projects/sendfile/sys/vm/vnode_pager.h Modified: projects/sendfile/sys/kern/vfs_default.c ============================================================================== --- projects/sendfile/sys/kern/vfs_default.c Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/kern/vfs_default.c Wed Feb 12 19:51:12 2014 (r261807) @@ -111,6 +111,7 @@ struct vop_vector default_vnodeops = { .vop_close = VOP_NULL, .vop_fsync = VOP_NULL, .vop_getpages = vop_stdgetpages, + .vop_getpages_async = vop_stdgetpages_async, .vop_getwritemount = vop_stdgetwritemount, .vop_inactive = VOP_NULL, .vop_ioctl = VOP_ENOTTY, @@ -726,7 +727,16 @@ vop_stdgetpages(ap) { return vnode_pager_generic_getpages(ap->a_vp, ap->a_m, - ap->a_count, ap->a_reqpage); + ap->a_count, ap->a_reqpage, NULL, NULL); +} + +/* XXX Needs good comment and a manpage. */ +int +vop_stdgetpages_async(struct vop_getpages_async_args *ap) +{ + + return vnode_pager_generic_getpages(ap->a_vp, ap->a_m, + ap->a_count, ap->a_reqpage, ap->a_vop_getpages_iodone, ap->a_arg); } int Modified: projects/sendfile/sys/kern/vnode_if.src ============================================================================== --- projects/sendfile/sys/kern/vnode_if.src Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/kern/vnode_if.src Wed Feb 12 19:51:12 2014 (r261807) @@ -477,6 +477,19 @@ vop_getpages { }; +%% getpages_async vp L L L + +vop_getpages_async { + IN struct vnode *vp; + IN vm_page_t *m; + IN int count; + IN int reqpage; + IN vm_ooffset_t offset; + IN void (*vop_getpages_iodone)(void *); + IN void *arg; +}; + + %% putpages vp L L L vop_putpages { Modified: projects/sendfile/sys/sys/vnode.h ============================================================================== --- projects/sendfile/sys/sys/vnode.h Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/sys/vnode.h Wed Feb 12 19:51:12 2014 (r261807) @@ -719,6 +719,7 @@ int vop_stdbmap(struct vop_bmap_args *); int vop_stdfsync(struct vop_fsync_args *); int vop_stdgetwritemount(struct vop_getwritemount_args *); int vop_stdgetpages(struct vop_getpages_args *); +int vop_stdgetpages_async(struct vop_getpages_async_args *); int vop_stdinactive(struct vop_inactive_args *); int vop_stdislocked(struct vop_islocked_args *); int vop_stdkqfilter(struct vop_kqfilter_args *); Modified: projects/sendfile/sys/ufs/ffs/ffs_vnops.c ============================================================================== --- projects/sendfile/sys/ufs/ffs/ffs_vnops.c Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/ufs/ffs/ffs_vnops.c Wed Feb 12 19:51:12 2014 (r261807) @@ -105,6 +105,7 @@ extern int ffs_rawread(struct vnode *vp, static vop_fsync_t ffs_fsync; static vop_lock1_t ffs_lock; static vop_getpages_t ffs_getpages; +static vop_getpages_async_t ffs_getpages_async; static vop_read_t ffs_read; static vop_write_t ffs_write; static int ffs_extread(struct vnode *vp, struct uio *uio, int ioflag); @@ -125,6 +126,7 @@ struct vop_vector ffs_vnodeops1 = { .vop_default = &ufs_vnodeops, .vop_fsync = ffs_fsync, .vop_getpages = ffs_getpages, + .vop_getpages_async = ffs_getpages_async, .vop_lock1 = ffs_lock, .vop_read = ffs_read, .vop_reallocblks = ffs_reallocblks, @@ -839,18 +841,16 @@ ffs_write(ap) } /* - * get page routine + * Get page routines. */ static int -ffs_getpages(ap) - struct vop_getpages_args *ap; +ffs_getpages_checkvalid(vm_page_t *m, int count, int reqpage) { - int i; vm_page_t mreq; int pcount; - pcount = round_page(ap->a_count) / PAGE_SIZE; - mreq = ap->a_m[ap->a_reqpage]; + pcount = round_page(count) / PAGE_SIZE; + mreq = m[reqpage]; /* * if ANY DEV_BSIZE blocks are valid on a large filesystem block, @@ -862,23 +862,47 @@ ffs_getpages(ap) if (mreq->valid) { if (mreq->valid != VM_PAGE_BITS_ALL) vm_page_zero_invalid(mreq, TRUE); - for (i = 0; i < pcount; i++) { - if (i != ap->a_reqpage) { - vm_page_lock(ap->a_m[i]); - vm_page_free(ap->a_m[i]); - vm_page_unlock(ap->a_m[i]); + for (int i = 0; i < pcount; i++) { + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_free(m[i]); + vm_page_unlock(m[i]); } } VM_OBJECT_WUNLOCK(mreq->object); - return VM_PAGER_OK; + return (VM_PAGER_OK); } VM_OBJECT_WUNLOCK(mreq->object); - return vnode_pager_generic_getpages(ap->a_vp, ap->a_m, - ap->a_count, - ap->a_reqpage); + return (-1); } +static int +ffs_getpages(struct vop_getpages_args *ap) +{ + int rv; + + rv = ffs_getpages_checkvalid(ap->a_m, ap->a_count, ap->a_reqpage); + if (rv == VM_PAGER_OK) + return (rv); + + return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, + ap->a_reqpage, NULL, NULL)); +} + +static int +ffs_getpages_async(struct vop_getpages_async_args *ap) +{ + int rv; + + rv = ffs_getpages_checkvalid(ap->a_m, ap->a_count, ap->a_reqpage); + if (rv == VM_PAGER_OK) { + (ap->a_vop_getpages_iodone)(ap->a_arg); + return (rv); + } + return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, ap->a_count, + ap->a_reqpage, ap->a_vop_getpages_iodone, ap->a_arg)); +} /* * Extended attribute area reading. Modified: projects/sendfile/sys/vm/vm_pager.h ============================================================================== --- projects/sendfile/sys/vm/vm_pager.h Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/vm/vm_pager.h Wed Feb 12 19:51:12 2014 (r261807) @@ -51,18 +51,21 @@ typedef vm_object_t pgo_alloc_t(void *, struct ucred *); typedef void pgo_dealloc_t(vm_object_t); typedef int pgo_getpages_t(vm_object_t, vm_page_t *, int, int); +typedef int pgo_getpages_async_t(vm_object_t, vm_page_t *, int, int, + void(*)(void *), void *); typedef void pgo_putpages_t(vm_object_t, vm_page_t *, int, int, int *); typedef boolean_t pgo_haspage_t(vm_object_t, vm_pindex_t, int *, int *); typedef void pgo_pageunswapped_t(vm_page_t); struct pagerops { - pgo_init_t *pgo_init; /* Initialize pager. */ - pgo_alloc_t *pgo_alloc; /* Allocate pager. */ - pgo_dealloc_t *pgo_dealloc; /* Disassociate. */ - pgo_getpages_t *pgo_getpages; /* Get (read) page. */ - pgo_putpages_t *pgo_putpages; /* Put (write) page. */ - pgo_haspage_t *pgo_haspage; /* Does pager have page? */ - pgo_pageunswapped_t *pgo_pageunswapped; + pgo_init_t *pgo_init; /* Initialize pager. */ + pgo_alloc_t *pgo_alloc; /* Allocate pager. */ + pgo_dealloc_t *pgo_dealloc; /* Disassociate. */ + pgo_getpages_t *pgo_getpages; /* Get (read) page. */ + pgo_getpages_async_t *pgo_getpages_async; /* Get page asyncly. */ + pgo_putpages_t *pgo_putpages; /* Put (write) page. */ + pgo_haspage_t *pgo_haspage; /* Query page. */ + pgo_pageunswapped_t *pgo_pageunswapped; }; extern struct pagerops defaultpagerops; @@ -103,6 +106,8 @@ vm_object_t vm_pager_allocate(objtype_t, void vm_pager_bufferinit(void); void vm_pager_deallocate(vm_object_t); static __inline int vm_pager_get_pages(vm_object_t, vm_page_t *, int, int); +static __inline int vm_pager_get_pages_async(vm_object_t, vm_page_t *, int, + int, void(*)(void *), void *); static __inline boolean_t vm_pager_has_page(vm_object_t, vm_pindex_t, int *, int *); void vm_pager_init(void); vm_object_t vm_pager_object_lookup(struct pagerlst *, void *); @@ -131,6 +136,27 @@ vm_pager_get_pages( return (r); } +static __inline int +vm_pager_get_pages_async(vm_object_t object, vm_page_t *m, int count, + int reqpage, void (*iodone)(void *), void *arg) +{ + int r; + + VM_OBJECT_ASSERT_WLOCKED(object); + + if (*pagertab[object->type]->pgo_getpages_async == NULL) { + /* Emulate async operation. */ + r = vm_pager_get_pages(object, m, count, reqpage); + VM_OBJECT_WUNLOCK(object); + (iodone)(arg); + VM_OBJECT_WLOCK(object); + } else + r = (*pagertab[object->type]->pgo_getpages_async)(object, m, + count, reqpage, iodone, arg); + + return (r); +} + static __inline void vm_pager_put_pages( vm_object_t object, Modified: projects/sendfile/sys/vm/vnode_pager.c ============================================================================== --- projects/sendfile/sys/vm/vnode_pager.c Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/vm/vnode_pager.c Wed Feb 12 19:51:12 2014 (r261807) @@ -83,6 +83,8 @@ static int vnode_pager_input_smlfs(vm_ob static int vnode_pager_input_old(vm_object_t object, vm_page_t m); static void vnode_pager_dealloc(vm_object_t); static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); +static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int, + void(*)(void *), void *); static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, boolean_t, int *); static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, @@ -92,6 +94,7 @@ struct pagerops vnodepagerops = { .pgo_alloc = vnode_pager_alloc, .pgo_dealloc = vnode_pager_dealloc, .pgo_getpages = vnode_pager_getpages, + .pgo_getpages_async = vnode_pager_getpages_async, .pgo_putpages = vnode_pager_putpages, .pgo_haspage = vnode_pager_haspage, }; @@ -664,17 +667,51 @@ vnode_pager_getpages(vm_object_t object, return rtval; } +static int +vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, + int reqpage, void (*iodone)(void *), void *arg) +{ + int rtval; + struct vnode *vp; + int bytes = count * PAGE_SIZE; + + vp = object->handle; + VM_OBJECT_WUNLOCK(object); + rtval = VOP_GETPAGES_ASYNC(vp, m, bytes, reqpage, 0, iodone, arg); + KASSERT(rtval != EOPNOTSUPP, + ("vnode_pager: FS getpages_async not implemented\n")); + VM_OBJECT_WLOCK(object); + return rtval; +} + +struct getpages_softc { + vm_page_t *m; + struct buf *bp; + vm_object_t object; + vm_offset_t kva; + off_t foff; + int size; + int count; + int unmapped; + int reqpage; + void (*iodone)(void *); + void *arg; +}; + +int vnode_pager_generic_getpages_done(struct getpages_softc *); +void vnode_pager_generic_getpages_done_async(struct buf *); + /* * This is now called from local media FS's to operate against their * own vnodes if they fail to implement VOP_GETPAGES. */ int vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, - int reqpage) + int reqpage, void (*iodone)(void *), void *arg) { vm_object_t object; vm_offset_t kva; - off_t foff, tfoff, nextoff; + off_t foff; int i, j, size, bsize, first; daddr_t firstaddr, reqblock; struct bufobj *bo; @@ -684,6 +721,7 @@ vnode_pager_generic_getpages(struct vnod struct mount *mp; int count; int error; + int unmapped; object = vp->v_object; count = bytecount / PAGE_SIZE; @@ -891,8 +929,8 @@ vnode_pager_generic_getpages(struct vnod * requires mapped buffers. */ mp = vp->v_mount; - if (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0 && - unmapped_buf_allowed) { + unmapped = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)); + if (unmapped && unmapped_buf_allowed) { bp->b_data = unmapped_buf; bp->b_kvabase = unmapped_buf; bp->b_offset = 0; @@ -905,7 +943,6 @@ vnode_pager_generic_getpages(struct vnod /* build a minimal buffer header */ bp->b_iocmd = BIO_READ; - bp->b_iodone = bdone; KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); bp->b_rcred = crhold(curthread->td_ucred); @@ -923,9 +960,89 @@ vnode_pager_generic_getpages(struct vnod /* do the input */ bp->b_iooffset = dbtob(bp->b_blkno); - bstrategy(bp); - bwait(bp, PVM, "vnread"); + if (iodone) { /* async */ + struct getpages_softc *sc; + + sc = malloc(sizeof(*sc), M_TEMP, M_WAITOK); + + sc->m = m; + sc->bp = bp; + sc->object = object; + sc->foff = foff; + sc->size = size; + sc->count = count; + sc->unmapped = unmapped; + sc->reqpage = reqpage; + sc->kva = kva; + + sc->iodone = iodone; + sc->arg = arg; + + bp->b_iodone = vnode_pager_generic_getpages_done_async; + bp->b_caller1 = sc; + BUF_KERNPROC(bp); + bstrategy(bp); + /* Good bye! */ + } else { + struct getpages_softc sc; + + sc.m = m; + sc.bp = bp; + sc.object = object; + sc.foff = foff; + sc.size = size; + sc.count = count; + sc.unmapped = unmapped; + sc.reqpage = reqpage; + sc.kva = kva; + + bp->b_iodone = bdone; + bstrategy(bp); + bwait(bp, PVM, "vnread"); + error = vnode_pager_generic_getpages_done(&sc); + } + + return (error ? VM_PAGER_ERROR : VM_PAGER_OK); +} + +void +vnode_pager_generic_getpages_done_async(struct buf *bp) +{ + struct getpages_softc *sc = bp->b_caller1; + int error; + + error = vnode_pager_generic_getpages_done(sc); + if (error) + printf("zhopa %d\n", error); + + vm_page_xunbusy(sc->m[sc->reqpage]); + + sc->iodone(sc->arg); + + free(sc, M_TEMP); +} + +int +vnode_pager_generic_getpages_done(struct getpages_softc *sc) +{ + vm_object_t object; + vm_offset_t kva; + vm_page_t *m; + struct buf *bp; + off_t foff, tfoff, nextoff; + int i, size, count, unmapped, reqpage; + int error = 0; + + m = sc->m; + bp = sc->bp; + object = sc->object; + foff = sc->foff; + size = sc->size; + count = sc->count; + unmapped = sc->unmapped; + reqpage = sc->reqpage; + kva = sc->kva; if ((bp->b_ioflags & BIO_ERROR) != 0) error = EIO; @@ -939,7 +1056,7 @@ vnode_pager_generic_getpages(struct vnod } if ((bp->b_flags & B_UNMAPPED) == 0) pmap_qremove(kva, count); - if (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) { + if (unmapped) { bp->b_data = (caddr_t)kva; bp->b_kvabase = (caddr_t)kva; bp->b_flags &= ~B_UNMAPPED; @@ -995,7 +1112,8 @@ vnode_pager_generic_getpages(struct vnod if (error) { printf("vnode_pager_getpages: I/O read error\n"); } - return (error ? VM_PAGER_ERROR : VM_PAGER_OK); + + return (error); } /* Modified: projects/sendfile/sys/vm/vnode_pager.h ============================================================================== --- projects/sendfile/sys/vm/vnode_pager.h Wed Feb 12 19:26:08 2014 (r261806) +++ projects/sendfile/sys/vm/vnode_pager.h Wed Feb 12 19:51:12 2014 (r261807) @@ -41,7 +41,7 @@ #ifdef _KERNEL int vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, - int count, int reqpage); + int count, int reqpage, void (*iodone)(void *), void *arg); int vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int count, boolean_t sync, int *rtvals);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201402121951.s1CJpDAc060029>