From owner-svn-src-projects@FreeBSD.ORG Tue Apr 24 00:14:00 2012 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 164E8106566B; Tue, 24 Apr 2012 00:14:00 +0000 (UTC) (envelope-from rmacklem@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id F38D78FC0A; Tue, 24 Apr 2012 00:13:59 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q3O0DxaZ021786; Tue, 24 Apr 2012 00:13:59 GMT (envelope-from rmacklem@svn.freebsd.org) Received: (from rmacklem@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q3O0Dxhj021779; Tue, 24 Apr 2012 00:13:59 GMT (envelope-from rmacklem@svn.freebsd.org) Message-Id: <201204240013.q3O0Dxhj021779@svn.freebsd.org> From: Rick Macklem Date: Tue, 24 Apr 2012 00:13:59 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r234626 - in projects/nfsv4.1-client/sys/fs: nfs nfsclient X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 24 Apr 2012 00:14:00 -0000 Author: rmacklem Date: Tue Apr 24 00:13:59 2012 New Revision: 234626 URL: http://svn.freebsd.org/changeset/base/234626 Log: Fix up layout recall, layout commit and layout return so that they seem to work ok. I'm still not clear w.r.t. exactly what RFC5661 requires them to do, so much more testing will be needed. Modified: projects/nfsv4.1-client/sys/fs/nfs/nfs_var.h projects/nfsv4.1-client/sys/fs/nfs/nfsclstate.h projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clbio.c projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clrpcops.c projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clstate.c projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clvnops.c projects/nfsv4.1-client/sys/fs/nfsclient/nfsnode.h Modified: projects/nfsv4.1-client/sys/fs/nfs/nfs_var.h ============================================================================== --- projects/nfsv4.1-client/sys/fs/nfs/nfs_var.h Mon Apr 23 23:57:24 2012 (r234625) +++ projects/nfsv4.1-client/sys/fs/nfs/nfs_var.h Tue Apr 24 00:13:59 2012 (r234626) @@ -450,11 +450,12 @@ int nfsrpc_layoutget(struct nfsmount *, struct ucred *, NFSPROC_T *, void *); int nfsrpc_getdeviceinfo(struct nfsmount *, uint8_t *, int, uint32_t *, struct nfscldevinfo **, struct ucred *, NFSPROC_T *); -int nfsrpc_layoutcommit(vnode_t, off_t, uint64_t, int, nfsv4stateid_t *, int, - off_t, int, struct timespec, int, int, uint8_t *, int *, uint64_t *, +int nfsrpc_layoutcommit(struct nfsmount *, uint8_t *, int, int, + uint64_t, uint64_t, nfsv4stateid_t *, int, int, uint8_t *, struct ucred *, NFSPROC_T *, void *); -int nfsrpc_layoutreturn(vnode_t, int, int, int, int, off_t, uint64_t, - nfsv4stateid_t *, int, uint32_t *, struct ucred *, NFSPROC_T *, void *); +int nfsrpc_layoutreturn(struct nfsmount *, uint8_t *, int, int, int, uint32_t, + int, uint64_t, uint64_t, nfsv4stateid_t *, int, uint32_t *, struct ucred *, + NFSPROC_T *, void *); int nfsrpc_reclaimcomplete(struct nfsmount *, struct ucred *, NFSPROC_T *); int nfscl_doiods(vnode_t, struct uio *, int *, int *, uint32_t, struct ucred *, NFSPROC_T *); @@ -520,17 +521,21 @@ void nfscl_deleggetmodtime(vnode_t, stru int nfscl_tryclose(struct nfsclopen *, struct ucred *, struct nfsmount *, NFSPROC_T *); void nfscl_cleanup(NFSPROC_T *); -int nfscl_layout(struct nfsmount *, u_int8_t *, int, nfsv4stateid_t *, int, - struct nfsclflayouthead *, struct nfscllayout **, struct ucred *, +int nfscl_layout(struct nfsmount *, vnode_t, u_int8_t *, int, nfsv4stateid_t *, + int, struct nfsclflayouthead *, struct nfscllayout **, struct ucred *, NFSPROC_T *); -struct nfscllayout *nfscl_getlayout(struct nfsclclient *, uint8_t *, int); +struct nfscllayout *nfscl_getlayout(struct nfsclclient *, uint8_t *, int, + int *); void nfscl_rellayout(struct nfscllayout *); -struct nfscldevinfo *nfscl_getdevinfo(struct nfsclclient *, uint8_t *); +struct nfscldevinfo *nfscl_getdevinfo(struct nfsclclient *, uint8_t *, + struct nfscldevinfo *); void nfscl_reldevinfo(struct nfscldevinfo *); -void nfscl_adddevinfo(struct nfsmount *, struct nfscldevinfo *); +int nfscl_adddevinfo(struct nfsmount *, struct nfscldevinfo *, + struct nfsclflayout *); void nfscl_freelayout(struct nfscllayout *); void nfscl_freeflayout(struct nfsclflayout *); void nfscl_freedevinfo(struct nfscldevinfo *); +int nfscl_layoutcommit(vnode_t, NFSPROC_T *); /* nfs_clport.c */ int nfscl_nget(mount_t, vnode_t, struct nfsfh *, Modified: projects/nfsv4.1-client/sys/fs/nfs/nfsclstate.h ============================================================================== --- projects/nfsv4.1-client/sys/fs/nfs/nfsclstate.h Mon Apr 23 23:57:24 2012 (r234625) +++ projects/nfsv4.1-client/sys/fs/nfs/nfsclstate.h Tue Apr 24 00:13:59 2012 (r234626) @@ -43,18 +43,14 @@ LIST_HEAD(nfscldeleghash, nfscldeleg); TAILQ_HEAD(nfscllayouthead, nfscllayout); LIST_HEAD(nfscllayouthash, nfscllayout); LIST_HEAD(nfsclflayouthead, nfsclflayout); -TAILQ_HEAD(nfscldevinfohead, nfscldevinfo); -LIST_HEAD(nfscldevinfohash, nfscldevinfo); +LIST_HEAD(nfscldevinfohead, nfscldevinfo); +LIST_HEAD(nfsclrecalllayouthead, nfsclrecalllayout); #define NFSCLDELEGHASHSIZE 256 #define NFSCLDELEGHASH(c, f, l) \ (&((c)->nfsc_deleghash[ncl_hash((f), (l)) % NFSCLDELEGHASHSIZE])) #define NFSCLLAYOUTHASHSIZE 256 #define NFSCLLAYOUTHASH(c, f, l) \ (&((c)->nfsc_layouthash[ncl_hash((f), (l)) % NFSCLLAYOUTHASHSIZE])) -#define NFSCLDEVINFOHASHSIZE 16 -#define NFSCLDEVINFOHASH(c, f) \ - (&((c)->nfsc_devinfohash[ncl_hash((f), NFSX_V4DEVICEID) % \ - NFSCLDEVINFOHASHSIZE])) /* Structure for NFSv4.1 session stuff. */ struct nfsclsession { @@ -100,7 +96,6 @@ struct nfsclclient { struct nfscllayouthead nfsc_layout; struct nfscllayouthash nfsc_layouthash[NFSCLLAYOUTHASHSIZE]; struct nfscldevinfohead nfsc_devinfo; - struct nfscldevinfohash nfsc_devinfohash[NFSCLDEVINFOHASHSIZE]; struct nfsv4lock nfsc_lock; struct proc *nfsc_renewthread; struct nfsmount *nfsc_nmp; @@ -234,16 +229,31 @@ struct nfscllayout { TAILQ_ENTRY(nfscllayout) nfsly_list; LIST_ENTRY(nfscllayout) nfsly_hash; nfsv4stateid_t nfsly_stateid; + uint64_t nfsly_filesid[2]; struct nfsclflayouthead nfsly_flayread; struct nfsclflayouthead nfsly_flayrw; + struct nfsclrecalllayouthead nfsly_recall; + time_t nfsly_timestamp; struct nfsclclient *nfsly_clp; uint32_t nfsly_refcnt; - uint16_t nfsly_retonclose; + uint16_t nfsly_flags; uint16_t nfsly_fhlen; uint8_t nfsly_fh[1]; }; /* + * Flags for nfsly_flags. + */ +#define NFSLY_FILES 0x0001 +#define NFSLY_BLOCK 0x0002 +#define NFSLY_OBJECT 0x0004 +#define NFSLY_RECALL 0x0008 +#define NFSLY_RECALLFILE 0x0010 +#define NFSLY_RECALLFSID 0x0020 +#define NFSLY_RECALLALL 0x0040 +#define NFSLY_RETONCLOSE 0x0080 + +/* * MALLOC'd to the correct length to accommodate the file handle list. * These hang off of nfsly_flayread and nfsly_flayrw, sorted in increasing * offset order. @@ -256,14 +266,34 @@ struct nfsclflayout { uint64_t nfsfl_off; uint64_t nfsfl_end; uint64_t nfsfl_patoff; + struct nfscldevinfo *nfsfl_devp; uint32_t nfsfl_iomode; uint32_t nfsfl_util; uint32_t nfsfl_stripe1; - uint32_t nfsfl_fhcnt; + uint16_t nfsfl_flags; + uint16_t nfsfl_fhcnt; struct nfsfh *nfsfl_fh[1]; /* FH list for DS */ }; /* + * Flags for nfsfl_flags. + */ +#define NFSFL_RECALL 0x0001 /* File layout has been recalled */ +#define NFSFL_WRITTEN 0x0002 /* Has been used to write to a DS. */ + +/* + * Structure that is used to store a LAYOUTRECALL. + */ +struct nfsclrecalllayout { + LIST_ENTRY(nfsclrecalllayout) nfsrecly_list; + uint64_t nfsrecly_off; + uint64_t nfsrecly_len; + int nfsrecly_recalltype; + uint32_t nfsrecly_iomode; + uint32_t nfsrecly_stateseqid; +}; + +/* * Stores the NFSv4.1 Device Info. Malloc'd to the correct length to * store the list of network connections and list of indices. * nfsdi_data[] is allocated the following way: @@ -273,11 +303,11 @@ struct nfsclflayout { * indices select which address.) */ struct nfscldevinfo { - TAILQ_ENTRY(nfscldevinfo) nfsdi_list; - LIST_ENTRY(nfscldevinfo) nfsdi_hash; + LIST_ENTRY(nfscldevinfo) nfsdi_list; uint8_t nfsdi_deviceid[NFSX_V4DEVICEID]; struct nfsclclient *nfsdi_clp; uint32_t nfsdi_refcnt; + uint32_t nfsdi_layoutrefs; uint16_t nfsdi_stripecnt; uint16_t nfsdi_addrcnt; struct nfsclds *nfsdi_data[0]; Modified: projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clbio.c ============================================================================== --- projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clbio.c Mon Apr 23 23:57:24 2012 (r234625) +++ projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clbio.c Tue Apr 24 00:13:59 2012 (r234626) @@ -1343,6 +1343,8 @@ ncl_vinvalbuf(struct vnode *vp, int flag goto out; error = vinvalbuf(vp, flags, 0, slptimeo); } + if (NFSHASPNFS(nmp)) + nfscl_layoutcommit(vp, td); mtx_lock(&np->n_mtx); if (np->n_directio_asyncwr == 0) np->n_flag &= ~NMODIFIED; Modified: projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clrpcops.c ============================================================================== --- projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clrpcops.c Mon Apr 23 23:57:24 2012 (r234625) +++ projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clrpcops.c Tue Apr 24 00:13:59 2012 (r234626) @@ -98,8 +98,9 @@ static int nfsrpc_locku(struct nfsrv_des u_int32_t, struct ucred *, NFSPROC_T *, int); static int nfsrpc_setaclrpc(vnode_t, struct ucred *, NFSPROC_T *, struct acl *, nfsv4stateid_t *, void *); -static int nfsrpc_getlayout(struct nfsmount *, struct nfsfh *, int, uint32_t *, - nfsv4stateid_t *, struct ucred *, NFSPROC_T *); +static int nfsrpc_getlayout(struct nfsmount *, vnode_t, struct nfsfh *, int, + uint32_t *, nfsv4stateid_t *, struct nfscllayout **, struct ucred *, + NFSPROC_T *); static int nfsrpc_fillsa(struct nfsmount *, struct sockaddr_storage *, struct nfsclds **, NFSPROC_T *); static void nfscl_initsessionslots(struct nfsclsession *); @@ -253,8 +254,6 @@ nfsrpc_open(vnode_t vp, int amode, struc struct nfsmount *nmp = VFSTONFS(vnode_mount(vp)); u_int32_t mode, clidrev; int ret, newone, error, expireret = 0, retrycnt; - int iomode; - nfsv4stateid_t stateid; /* * For NFSv4, Open Ops are only done on Regular Files. @@ -262,13 +261,10 @@ nfsrpc_open(vnode_t vp, int amode, struc if (vnode_vtype(vp) != VREG) return (0); mode = 0; - iomode = NFSLAYOUTIOMODE_READ; if (amode & FREAD) mode |= NFSV4OPEN_ACCESSREAD; - if (amode & FWRITE) { + if (amode & FWRITE) mode |= NFSV4OPEN_ACCESSWRITE; - iomode = NFSLAYOUTIOMODE_RW; - } nfhp = np->n_fhp; retrycnt = 0; @@ -320,17 +316,6 @@ else printf(" fhl=0\n"); op->nfso_own->nfsow_clp, nfhp->nfh_fh, nfhp->nfh_len, cred, p, &dp); } - - /* Try and get a Layout, if it is supported. */ - if (error == 0 && NFSHASPNFS(nmp) && - nfscl_enablecallb != 0 && nfs_numnfscbd > 0) { - stateid.seqid = op->nfso_stateid.seqid; - stateid.other[0] = op->nfso_stateid.other[0]; - stateid.other[1] = op->nfso_stateid.other[1]; - stateid.other[2] = op->nfso_stateid.other[2]; - (void)nfsrpc_getlayout(nmp, nfhp, iomode, - NULL, &stateid, cred, p); - } } else { error = EIO; } @@ -4450,9 +4435,10 @@ printf("servlen=%d\n", len); dsp->nfsclds_sess.nfsess_sequenceid = fxdr_unsigned(uint32_t, *tl++); v41flags = fxdr_unsigned(uint32_t, *tl); -printf("v41fl=0x%x\n", v41flags); +printf("v41fl=0x%x nmfl=0x%x\n", v41flags, nmp->nm_flag); if ((v41flags & NFSV4EXCH_USEPNFSMDS) != 0 && NFSHASPNFSOPT(nmp)) { +printf("set PNFS\n"); NFSLOCKMNT(nmp); nmp->nm_state |= NFSSTA_PNFS; NFSUNLOCKMNT(nmp); @@ -4651,7 +4637,8 @@ nfsrpc_layoutget(struct nfsmount *nmp, u tl += 2; txdr_hyper(minlen, tl); tl += 2; - *tl++ = stateidp->seqid; + *tl++ = txdr_unsigned(stateidp->seqid); +printf("layget seq=%d\n", stateidp->seqid); *tl++ = stateidp->other[0]; *tl++ = stateidp->other[1]; *tl++ = stateidp->other[2]; @@ -4667,8 +4654,8 @@ nfsrpc_layoutget(struct nfsmount *nmp, u *retonclosep = 1; else *retonclosep = 0; -printf("retonclose=%d\n", *retonclosep); - stateidp->seqid = *tl++; + stateidp->seqid = fxdr_unsigned(uint32_t, *tl++); +printf("retoncls=%d stseq=%d\n", *retonclosep, stateidp->seqid); stateidp->other[0] = *tl++; stateidp->other[1] = *tl++; stateidp->other[2] = *tl++; @@ -4698,6 +4685,7 @@ printf("fhcnt=%d\n", fhcnt); else flp = malloc(sizeof(*flp), M_NFSFLAYOUT, M_WAITOK); + flp->nfsfl_flags = 0; flp->nfsfl_fhcnt = 0; flp->nfsfl_off = fxdr_hyper(tl); tl += 2; retlen = fxdr_hyper(tl); tl += 2; @@ -4934,21 +4922,20 @@ nfsmout: * Do the NFSv4.1 LayoutCommit. */ int -nfsrpc_layoutcommit(vnode_t vp, off_t offset, uint64_t len, int reclaim, - nfsv4stateid_t *stateidp, int newoff, off_t newoffset, int newtime, - struct timespec newtimespec, int layouttype, int layoutupdatecnt, - uint8_t *layp, int *sizechangedp, uint64_t *newsizep, struct ucred *cred, - NFSPROC_T *p, void *stuff) +nfsrpc_layoutcommit(struct nfsmount *nmp, uint8_t *fh, int fhlen, int reclaim, + uint64_t off, uint64_t len, nfsv4stateid_t *stateidp, int layouttype, + int layoutupdatecnt, uint8_t *layp, struct ucred *cred, NFSPROC_T *p, + void *stuff) { uint32_t *tl; struct nfsrv_descript nfsd, *nd = &nfsd; int error, outcnt, i; uint8_t *cp; - NFSCL_REQSTART(nd, NFSPROC_LAYOUTCOMMIT, vp); - NFSM_BUILD(tl, uint32_t *, 2 * NFSX_UNSIGNED + 2 * NFSX_HYPER + + nfscl_reqstart(nd, NFSPROC_LAYOUTCOMMIT, nmp, fh, fhlen, NULL, NULL); + NFSM_BUILD(tl, uint32_t *, 5 * NFSX_UNSIGNED + 2 * NFSX_HYPER + NFSX_STATEID); - txdr_hyper(offset, tl); + txdr_hyper(off, tl); tl += 2; txdr_hyper(len, tl); tl += 2; @@ -4956,25 +4943,12 @@ nfsrpc_layoutcommit(vnode_t vp, off_t of *tl++ = newnfs_true; else *tl++ = newnfs_false; - *tl++ = stateidp->seqid; + *tl++ = txdr_unsigned(stateidp->seqid); *tl++ = stateidp->other[0]; *tl++ = stateidp->other[1]; *tl++ = stateidp->other[2]; - if (newoff != 0) { - *tl = newnfs_true; - NFSM_BUILD(tl, uint32_t *, NFSX_HYPER); - txdr_hyper(newoffset, tl); - } else - *tl = newnfs_false; - if (newtime != 0) { - NFSM_BUILD(tl, uint32_t *, NFSX_V4SETTIME + 2 * NFSX_UNSIGNED); - *tl++ = newnfs_true; - txdr_nfsv4time(&newtimespec, tl); - tl += NFSX_V4TIME / NFSX_UNSIGNED; - } else { - NFSM_BUILD(tl, uint32_t *, 3 * NFSX_UNSIGNED); - *tl++ = newnfs_false; - } + *tl++ = newnfs_false; + *tl++ = newnfs_false; *tl++ = txdr_unsigned(layouttype); *tl = txdr_unsigned(layoutupdatecnt); if (layoutupdatecnt > 0) { @@ -4988,20 +4962,11 @@ nfsrpc_layoutcommit(vnode_t vp, off_t of *cp++ = 0x0; } nd->nd_flag |= ND_USEGSSNAME; - error = nfscl_request(nd, vp, p, cred, stuff); + error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred, + NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL); if (error != 0) return (error); - if (nd->nd_repstat == 0) { - NFSM_DISSECT(tl, uint32_t *, NFSX_UNSIGNED); - if (*tl != 0) { - NFSM_DISSECT(tl, uint32_t *, NFSX_HYPER); - *sizechangedp = 1; - *newsizep = fxdr_hyper(tl); - } else - *sizechangedp = 0; - } else - error = nd->nd_repstat; -nfsmout: + error = nd->nd_repstat; mbuf_freem(nd->nd_mrep); return (error); } @@ -5010,9 +4975,9 @@ nfsmout: * Do the NFSv4.1 LayoutReturn. */ int -nfsrpc_layoutreturn(vnode_t vp, int reclaim, int layouttype, int iomode, - int layoutreturn, off_t offset, uint64_t len, nfsv4stateid_t *stateidp, - int layoutcnt, uint32_t *layp, +nfsrpc_layoutreturn(struct nfsmount *nmp, uint8_t *fh, int fhlen, int reclaim, + int layouttype, uint32_t iomode, int layoutreturn, uint64_t offset, + uint64_t len, nfsv4stateid_t *stateidp, int layoutcnt, uint32_t *layp, struct ucred *cred, NFSPROC_T *p, void *stuff) { uint32_t *tl; @@ -5020,7 +4985,7 @@ nfsrpc_layoutreturn(vnode_t vp, int recl int error, outcnt, i; uint8_t *cp; - NFSCL_REQSTART(nd, NFSPROC_LAYOUTRETURN, vp); + nfscl_reqstart(nd, NFSPROC_LAYOUTRETURN, nmp, fh, fhlen, NULL, NULL); NFSM_BUILD(tl, uint32_t *, 4 * NFSX_UNSIGNED); if (reclaim != 0) *tl++ = newnfs_true; @@ -5036,7 +5001,8 @@ nfsrpc_layoutreturn(vnode_t vp, int recl tl += 2; txdr_hyper(len, tl); tl += 2; - *tl++ = stateidp->seqid; +printf("layret stseq=%d\n", stateidp->seqid); + *tl++ = txdr_unsigned(stateidp->seqid); *tl++ = stateidp->other[0]; *tl++ = stateidp->other[1]; *tl++ = stateidp->other[2]; @@ -5051,14 +5017,15 @@ nfsrpc_layoutreturn(vnode_t vp, int recl } } nd->nd_flag |= ND_USEGSSNAME; - error = nfscl_request(nd, vp, p, cred, stuff); + error = newnfs_request(nd, nmp, NULL, &nmp->nm_sockreq, NULL, p, cred, + NFS_PROG, NFS_VER4, NULL, 1, NULL, NULL); if (error != 0) return (error); if (nd->nd_repstat == 0) { NFSM_DISSECT(tl, uint32_t *, NFSX_UNSIGNED); if (*tl != 0) { NFSM_DISSECT(tl, uint32_t *, NFSX_STATEID); - stateidp->seqid = *tl++; + stateidp->seqid = fxdr_unsigned(uint32_t, *tl++); stateidp->other[0] = *tl++; stateidp->other[1] = *tl++; stateidp->other[2] = *tl; @@ -5071,42 +5038,58 @@ nfsmout: } /* - * Called from nfsrpc_open() to acquire a layout and associated device - * info. A separate function mostly to avoid excessive indentation in - * nfsrpc_open(). The open has already acquired a reference cnt on the client. + * Acquire a layout and devinfo, if possible. The caller must have acquired + * a reference count on the nfsclclient structure before calling this. + * Return the layout in lypp with a reference count on it, if successful. */ static int -nfsrpc_getlayout(struct nfsmount *nmp, struct nfsfh *nfhp, int iomode, - uint32_t *notifybitsp, nfsv4stateid_t *stateidp, struct ucred *cred, - NFSPROC_T *p) +nfsrpc_getlayout(struct nfsmount *nmp, vnode_t vp, struct nfsfh *nfhp, + int iomode, uint32_t *notifybitsp, nfsv4stateid_t *stateidp, + struct nfscllayout **lypp, struct ucred *cred, NFSPROC_T *p) { struct nfscllayout *lyp; struct nfsclflayout *flp; struct nfscldevinfo *dip; struct nfsclflayouthead flh; - int error = 0, retonclose; + int error = 0, recalled, retonclose; - lyp = nfscl_getlayout(nmp->nm_clp, nfhp->nfh_fh, nfhp->nfh_len); + *lypp = NULL; + lyp = nfscl_getlayout(nmp->nm_clp, nfhp->nfh_fh, nfhp->nfh_len, + &recalled); if (lyp == NULL) { + if (recalled != 0) + return (EIO); LIST_INIT(&flh); error = nfsrpc_layoutget(nmp, nfhp->nfh_fh, nfhp->nfh_len, iomode, (uint64_t)0, INT64_MAX, (uint64_t)0, stateidp, &retonclose, &flh, cred, p, NULL); if (error == 0) LIST_FOREACH(flp, &flh, nfsfl_list) { - error = nfsrpc_getdeviceinfo(nmp, - flp->nfsfl_dev, NFSLAYOUT_NFSV4_1_FILES, - notifybitsp, &dip, cred, p); - if (error != 0) - break; - nfscl_adddevinfo(nmp, dip); + error = nfscl_adddevinfo(nmp, NULL, flp); + if (error != 0) { + error = nfsrpc_getdeviceinfo(nmp, + flp->nfsfl_dev, + NFSLAYOUT_NFSV4_1_FILES, + notifybitsp, &dip, cred, p); + if (error != 0) + break; + error = nfscl_adddevinfo(nmp, dip, flp); + if (error != 0) + printf( + "getlayout: cannot add\n"); + } } if (error == 0) - error = nfscl_layout(nmp, nfhp->nfh_fh, nfhp->nfh_len, - stateidp, retonclose, &flh, &lyp, cred, p); + error = nfscl_layout(nmp, vp, nfhp->nfh_fh, + nfhp->nfh_len, stateidp, retonclose, &flh, &lyp, + cred, p); + } + if (lyp != NULL) { + if (error == 0) + *lypp = lyp; + else + nfscl_rellayout(lyp); } - if (lyp != NULL) - nfscl_rellayout(lyp); return (error); } @@ -5303,33 +5286,49 @@ nfscl_doiods(vnode_t vp, struct uio *uio nfsv4stateid_t stateid; struct ucred *newcred; uint64_t len, off, oresid, xfer; - int eof, error; + int eof, error, iolaymode, recalled; void *lckp; - if (!NFSHASPNFS(nmp) || nfscl_enablecallb == 0 || nfs_numnfscbd == 0) + if (!NFSHASPNFS(nmp) || nfscl_enablecallb == 0 || nfs_numnfscbd == 0 || + (np->n_flag & NNOLAYOUT) != 0) return (EIO); /* Now, get a reference cnt on the clientid for this mount. */ if (nfscl_getref(nmp) == 0) return (EIO); - /* Search for a layout for this file. */ - layp = nfscl_getlayout(nmp->nm_clp, np->n_fhp->nfh_fh, - np->n_fhp->nfh_len); - if (layp == NULL) { - nfscl_relref(nmp); - return (EIO); - } - /* Find an appropriate stateid. */ newcred = NFSNEWCRED(cred); error = nfscl_getstateid(vp, np->n_fhp->nfh_fh, np->n_fhp->nfh_len, rwaccess, 1, newcred, p, &stateid, &lckp); if (error != 0) { +if (error == 2) printf("rwacc=0x%x\n", rwaccess); NFSFREECRED(newcred); - nfscl_rellayout(layp); nfscl_relref(nmp); return (error); } + /* Search for a layout for this file. */ + layp = nfscl_getlayout(nmp->nm_clp, np->n_fhp->nfh_fh, + np->n_fhp->nfh_len, &recalled); + if (layp == NULL) { + /* Try and get a Layout, if it is supported. */ + stateid.seqid = 0; + iolaymode = (rwaccess == NFSV4OPEN_ACCESSWRITE) ? + NFSLAYOUTIOMODE_RW : NFSLAYOUTIOMODE_READ; + error = nfsrpc_getlayout(nmp, vp, np->n_fhp, iolaymode, + NULL, &stateid, &layp, newcred, p); + if (error != 0) { + NFSLOCKNODE(np); + np->n_flag |= NNOLAYOUT; + NFSUNLOCKNODE(np); + if (lckp != NULL) + nfscl_lockderef(lckp); + NFSFREECRED(newcred); + if (layp != NULL) + nfscl_rellayout(layp); + nfscl_relref(nmp); + return (error); + } + } /* * Loop around finding a layout that works for the first part of @@ -5345,7 +5344,8 @@ nfscl_doiods(vnode_t vp, struct uio *uio oresid = xfer = (uint64_t)uiop->uio_resid; if (xfer > (rflp->nfsfl_end - rflp->nfsfl_off)) xfer = rflp->nfsfl_end - rflp->nfsfl_off; - dip = nfscl_getdevinfo(nmp->nm_clp, rflp->nfsfl_dev); + dip = nfscl_getdevinfo(nmp->nm_clp, rflp->nfsfl_dev, + rflp->nfsfl_devp); if (dip != NULL) { error = nfscl_doflayoutio(vp, uiop, iomode, must_commit, &eof, &stateid, rwaccess, dip, @@ -5463,10 +5463,16 @@ nfscl_doflayoutio(vnode_t vp, struct uio if (rwflag == FREAD) error = nfsrpc_readds(vp, uiop, stateidp, eofp, *dspp, io_off, xfer, fhp, cred, p); - else + else { error = nfsrpc_writeds(vp, uiop, iomode, must_commit, stateidp, *dspp, io_off, xfer, fhp, commit_thru_mds, cred, p); + if (error == 0) { + NFSLOCKCLSTATE(); + flp->nfsfl_flags |= NFSFL_WRITTEN; + NFSUNLOCKCLSTATE(); + } + } if (error == 0) { transfer = stripe_unit_size; stripe_pos = (stripe_pos + 1) % dp->nfsdi_stripecnt; Modified: projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clstate.c ============================================================================== --- projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clstate.c Mon Apr 23 23:57:24 2012 (r234625) +++ projects/nfsv4.1-client/sys/fs/nfsclient/nfs_clstate.c Tue Apr 24 00:13:59 2012 (r234626) @@ -156,6 +156,13 @@ static void nfscl_emptylockowner(struct struct nfscllockownerfhhead *); static void nfscl_mergeflayouts(struct nfsclflayouthead *, struct nfsclflayouthead *); +static int nfscl_layoutrecall(int, struct nfscllayout *, uint32_t, uint64_t, + uint64_t, uint32_t, struct nfsclrecalllayout *); +static int nfscl_seq(uint32_t, uint32_t); +static void nfscl_layoutreturn(struct nfsmount *, struct nfscllayout *, + struct ucred *, NFSPROC_T *); +static void nfscl_dolayoutcommit(struct nfsmount *, struct nfscllayout *, + struct nfsclflayout *, struct ucred *, NFSPROC_T *); static short nfscberr_null[] = { 0, @@ -762,13 +769,11 @@ nfscl_getcl(struct mount *mp, struct ucr LIST_INIT(&clp->nfsc_owner); TAILQ_INIT(&clp->nfsc_deleg); TAILQ_INIT(&clp->nfsc_layout); - TAILQ_INIT(&clp->nfsc_devinfo); + LIST_INIT(&clp->nfsc_devinfo); for (i = 0; i < NFSCLDELEGHASHSIZE; i++) LIST_INIT(&clp->nfsc_deleghash[i]); for (i = 0; i < NFSCLLAYOUTHASHSIZE; i++) LIST_INIT(&clp->nfsc_layouthash[i]); - for (i = 0; i < NFSCLDEVINFOHASHSIZE; i++) - LIST_INIT(&clp->nfsc_devinfohash[i]); clp->nfsc_flags = NFSCLFLAGS_INITED; clp->nfsc_clientidrev = 1; clp->nfsc_cbident = nfscl_nextcbident(); @@ -2432,6 +2437,10 @@ nfscl_renewthread(struct nfsclclient *cl static time_t prevsec = 0; struct nfscllockownerfh *lfhp, *nlfhp; struct nfscllockownerfhhead lfh; + struct nfscllayout *lyp, *nlyp; + struct nfsclflayout *flp; + struct nfscldevinfo *dip, *ndip; + struct nfscllayouthead rlh; cred = newnfs_getcred(); NFSLOCKCLSTATE(); @@ -2577,8 +2586,90 @@ tryagain: } if (igotlock) nfsv4_unlock(&clp->nfsc_lock, 0); + + /* + * Do the recall on any layouts. To avoid trouble, always + * come back up here after having slept. + */ + TAILQ_INIT(&rlh); +tryagain2: + TAILQ_FOREACH_SAFE(lyp, &clp->nfsc_layout, nfsly_list, nlyp) { + if ((lyp->nfsly_flags & NFSLY_RECALL) != 0) { + /* + * Wait for outstanding I/O ops to be done. + */ + if (lyp->nfsly_refcnt > 0) { +printf("layrec io=%d\n", lyp->nfsly_refcnt); + (void)mtx_sleep(&lyp->nfsly_refcnt, + NFSCLSTATEMUTEXPTR, PZERO, "nfslyd", + 0); + goto tryagain2; + } + + /* Handle any layout commits. */ + LIST_FOREACH(flp, &lyp->nfsly_flayrw, + nfsfl_list) { + if ((flp->nfsfl_flags & NFSFL_WRITTEN) + != 0) { + lyp->nfsly_refcnt++; + flp->nfsfl_flags &= + ~NFSFL_WRITTEN; + NFSUNLOCKCLSTATE(); +printf("do layoutcommit\n"); + nfscl_dolayoutcommit( + clp->nfsc_nmp, lyp, flp, + cred, p); + NFSLOCKCLSTATE(); + lyp->nfsly_refcnt--; + if (lyp->nfsly_refcnt == 0) + wakeup( + &lyp->nfsly_refcnt); + goto tryagain2; + } + } + + /* Move the layout to the recall list. */ + TAILQ_REMOVE(&clp->nfsc_layout, lyp, + nfsly_list); + LIST_REMOVE(lyp, nfsly_hash); + TAILQ_INSERT_HEAD(&rlh, lyp, nfsly_list); + } + } +#ifdef notyet + /* Now, look for stale layouts. */ + lyp = TAILQ_LAST(&clp->nfsc_layout, nfscllayouthead); + while (lyp != NULL) { + nlyp = TAILQ_PREV(lyp, nfscllayouthead, nfsly_list); + if (lyp->nfsly_timestamp < NFSD_MONOSEC && + (lyp->nfsly_flags & NFSLY_RECALL) == 0 && + lyp->nfsly_refcnt == 0) { +printf("ret stale lay=%d\n", nfscl_layoutcnt); + } + lyp = nlyp; + } +#endif + + /* + * Free up any unreferenced device info structures. + */ + LIST_FOREACH_SAFE(dip, &clp->nfsc_devinfo, nfsdi_list, ndip) { + if (dip->nfsdi_layoutrefs == 0 && + dip->nfsdi_refcnt == 0) { +printf("freeing devinfo\n"); + LIST_REMOVE(dip, nfsdi_list); + nfscl_freedevinfo(dip); + } + } NFSUNLOCKCLSTATE(); + /* Do layout return(s), as required. */ + TAILQ_FOREACH_SAFE(lyp, &rlh, nfsly_list, nlyp) { + TAILQ_REMOVE(&rlh, lyp, nfsly_list); +printf("ret layout\n"); + nfscl_layoutreturn(clp->nfsc_nmp, lyp, cred, p); + nfscl_freelayout(lyp); + } + /* * Delegreturn any delegations cleaned out or recalled. */ @@ -3031,6 +3122,11 @@ nfscl_docb(struct nfsrv_descript *nd, NF uint32_t seqid, slotid = 0, highslot, cachethis; uint8_t sessionid[NFSX_V4SESSIONID]; struct mbuf *rep; + struct nfscllayout *lyp; + uint64_t filesid[2], len, off; + int changed, gotone, laytype, recalltype; + uint32_t iomode; + struct nfsclrecalllayout *recallp = NULL; gotseq_ok = 0; nfsrvd_rephead(nd); @@ -3186,6 +3282,129 @@ printf("cbrecall\n"); if (nfhp != NULL) FREE((caddr_t)nfhp, M_NFSFH); break; + case NFSV4OP_CBLAYOUTRECALL: +printf("cblayrec\n"); + nfhp = NULL; + NFSM_DISSECT(tl, uint32_t *, 4 * NFSX_UNSIGNED); + laytype = fxdr_unsigned(int, *tl++); + iomode = fxdr_unsigned(uint32_t, *tl++); + if (newnfs_true == *tl++) + changed = 1; + else + changed = 0; + recalltype = fxdr_unsigned(int, *tl); + recallp = malloc(sizeof(*recallp), M_NFSLAYRECALL, + M_WAITOK); + if (laytype != NFSLAYOUT_NFSV4_1_FILES) + error = NFSERR_NOMATCHLAYOUT; + else if (recalltype == NFSLAYOUTRETURN_FILE) { + error = nfsm_getfh(nd, &nfhp); +printf("retfile getfh=%d\n", error); + if (error != 0) + goto nfsmout; + NFSM_DISSECT(tl, u_int32_t *, 2 * NFSX_HYPER + + NFSX_STATEID); + off = fxdr_hyper(tl); tl += 2; + len = fxdr_hyper(tl); tl += 2; + stateid.seqid = fxdr_unsigned(uint32_t, *tl++); + NFSBCOPY(tl, stateid.other, NFSX_STATEIDOTHER); + if (minorvers == NFSV4_MINORVERSION) + error = NFSERR_NOTSUPP; + else if (i == 0) + error = NFSERR_OPNOTINSESS; + if (error == 0) { + NFSLOCKCLSTATE(); + clp = nfscl_getclntsess(sessionid); +printf("cbly clp=%p\n", clp); + if (clp != NULL) { + lyp = nfscl_findlayout(clp, + nfhp->nfh_fh, + nfhp->nfh_len); +printf("cblyp=%p\n", lyp); + if (lyp != NULL && + (lyp->nfsly_flags & + NFSLY_FILES) != 0 && + !NFSBCMP(stateid.other, + lyp->nfsly_stateid.other, + NFSX_STATEIDOTHER)) { + error = + nfscl_layoutrecall( + recalltype, + lyp, iomode, off, + len, stateid.seqid, + recallp); + recallp = NULL; + wakeup(clp); +printf("aft layrec=%d\n", error); + } else + error = + NFSERR_NOMATCHLAYOUT; + } else + error = NFSERR_NOMATCHLAYOUT; + NFSUNLOCKCLSTATE(); + } + free(nfhp, M_NFSFH); + } else if (recalltype == NFSLAYOUTRETURN_FSID) { + NFSM_DISSECT(tl, uint32_t *, 2 * NFSX_HYPER); + filesid[0] = fxdr_hyper(tl); tl += 2; + filesid[1] = fxdr_hyper(tl); tl += 2; + gotone = 0; + NFSLOCKCLSTATE(); + clp = nfscl_getclntsess(sessionid); + if (clp != NULL) { + TAILQ_FOREACH(lyp, &clp->nfsc_layout, + nfsly_list) { + if (lyp->nfsly_filesid[0] == + filesid[0] && + lyp->nfsly_filesid[1] == + filesid[1]) { + error = + nfscl_layoutrecall( + recalltype, + lyp, iomode, 0, + UINT64_MAX, + lyp->nfsly_stateid.seqid, + recallp); + recallp = NULL; + gotone = 1; + } + } + if (gotone != 0) + wakeup(clp); + else + error = NFSERR_NOMATCHLAYOUT; + } else + error = NFSERR_NOMATCHLAYOUT; + NFSUNLOCKCLSTATE(); + } else if (recalltype == NFSLAYOUTRETURN_ALL) { + gotone = 0; + NFSLOCKCLSTATE(); + clp = nfscl_getclntsess(sessionid); + if (clp != NULL) { + TAILQ_FOREACH(lyp, &clp->nfsc_layout, + nfsly_list) { + error = nfscl_layoutrecall( + recalltype, lyp, iomode, 0, + UINT64_MAX, + lyp->nfsly_stateid.seqid, + recallp); + recallp = NULL; + gotone = 1; + } + if (gotone != 0) + wakeup(clp); + else + error = NFSERR_NOMATCHLAYOUT; + } else + error = NFSERR_NOMATCHLAYOUT; + NFSUNLOCKCLSTATE(); + } else + error = NFSERR_NOMATCHLAYOUT; + if (recallp != NULL) { + free(recallp, M_NFSLAYRECALL); + recallp = NULL; + } + break; case NFSV4OP_CBSEQUENCE: NFSM_DISSECT(tl, uint32_t *, NFSX_V4SESSIONID + 5 * NFSX_UNSIGNED); @@ -3267,6 +3486,8 @@ printf("unsupp callback %d\n", op); *repp = 0; /* NFS4_OK */ } nfsmout: + if (recallp != NULL) + free(recallp, M_NFSLAYRECALL); if (error) { if (error == EBADRPC || error == NFSERR_BADXDR) nd->nd_repstat = NFSERR_BADXDR; @@ -4413,7 +4634,7 @@ nfscl_errmap(struct nfsrv_descript *nd) * Called to find/add a layout to a client. */ APPLESTATIC int -nfscl_layout(struct nfsmount *nmp, u_int8_t *fhp, int fhlen, +nfscl_layout(struct nfsmount *nmp, vnode_t vp, u_int8_t *fhp, int fhlen, nfsv4stateid_t *stateidp, int retonclose, struct nfsclflayouthead *fhlp, struct nfscllayout **lypp, struct ucred *cred, NFSPROC_T *p) @@ -4421,6 +4642,7 @@ nfscl_layout(struct nfsmount *nmp, u_int struct nfsclclient *clp; struct nfscllayout *lyp, *tlyp; struct nfsclflayout *flp; + struct nfsnode *np = VTONFS(vp); *lypp = NULL; tlyp = malloc(sizeof(*tlyp) + fhlen - 1, M_NFSLAYOUT, M_WAITOK); @@ -4442,22 +4664,27 @@ nfscl_layout(struct nfsmount *nmp, u_int lyp->nfsly_stateid.other[2] = stateidp->other[2]; LIST_INIT(&lyp->nfsly_flayread); LIST_INIT(&lyp->nfsly_flayrw); + LIST_INIT(&lyp->nfsly_recall); + lyp->nfsly_filesid[0] = np->n_vattr.na_filesid[0]; + lyp->nfsly_filesid[1] = np->n_vattr.na_filesid[1]; lyp->nfsly_clp = clp; - lyp->nfsly_retonclose = retonclose; + lyp->nfsly_flags = (retonclose != 0) ? + (NFSLY_FILES | NFSLY_RETONCLOSE) : NFSLY_FILES; lyp->nfsly_refcnt = 1; /* Return with a reference cnt. */ lyp->nfsly_fhlen = fhlen; NFSBCOPY(fhp, lyp->nfsly_fh, fhlen); TAILQ_INSERT_HEAD(&clp->nfsc_layout, lyp, nfsly_list); LIST_INSERT_HEAD(NFSCLLAYOUTHASH(clp, fhp, fhlen), lyp, nfsly_hash); -#ifdef notyet lyp->nfsly_timestamp = NFSD_MONOSEC + 120; -#endif nfscl_layoutcnt++; } else { lyp->nfsly_refcnt++; + if (retonclose != 0) + lyp->nfsly_flags |= NFSLY_RETONCLOSE; TAILQ_REMOVE(&clp->nfsc_layout, lyp, nfsly_list); TAILQ_INSERT_HEAD(&clp->nfsc_layout, lyp, nfsly_list); + lyp->nfsly_timestamp = NFSD_MONOSEC + 120; } /* Merge the new list of File Layouts into the list. */ @@ -4476,20 +4703,27 @@ nfscl_layout(struct nfsmount *nmp, u_int } /* - * Search for a layout by MDS file handle. If one is found, lock it it and + * Search for a layout by MDS file handle. If one is found, ref cnt it and * return a pointer to it. */ struct nfscllayout * -nfscl_getlayout(struct nfsclclient *clp, uint8_t *fhp, int fhlen) +nfscl_getlayout(struct nfsclclient *clp, uint8_t *fhp, int fhlen, + int *recalledp) { struct nfscllayout *lyp; + *recalledp = 0; NFSLOCKCLSTATE(); lyp = nfscl_findlayout(clp, fhp, fhlen); if (lyp != NULL) { - lyp->nfsly_refcnt++; - TAILQ_REMOVE(&clp->nfsc_layout, lyp, nfsly_list); - TAILQ_INSERT_HEAD(&clp->nfsc_layout, lyp, nfsly_list); + if ((lyp->nfsly_flags & NFSLY_RECALL) == 0) { + lyp->nfsly_refcnt++; + TAILQ_REMOVE(&clp->nfsc_layout, lyp, nfsly_list); + TAILQ_INSERT_HEAD(&clp->nfsc_layout, lyp, nfsly_list); + } else { + lyp = NULL; + *recalledp = 1; + } } NFSUNLOCKCLSTATE(); return (lyp); @@ -4514,17 +4748,15 @@ nfscl_rellayout(struct nfscllayout *lyp) * acquiring a reference count on it. */ struct nfscldevinfo * -nfscl_getdevinfo(struct nfsclclient *clp, uint8_t *deviceid) +nfscl_getdevinfo(struct nfsclclient *clp, uint8_t *deviceid, + struct nfscldevinfo *dip) { - struct nfscldevinfo *dip; NFSLOCKCLSTATE(); - dip = nfscl_finddevinfo(clp, deviceid); - if (dip != NULL) { + if (dip == NULL) + dip = nfscl_finddevinfo(clp, deviceid); + if (dip != NULL) dip->nfsdi_refcnt++; - TAILQ_REMOVE(&clp->nfsc_devinfo, dip, nfsdi_list); - TAILQ_INSERT_HEAD(&clp->nfsc_devinfo, dip, nfsdi_list); - } NFSUNLOCKCLSTATE(); return (dip); } @@ -4576,7 +4808,7 @@ nfscl_finddevinfo(struct nfsclclient *cl { struct nfscldevinfo *dip; - LIST_FOREACH(dip, NFSCLDEVINFOHASH(clp, deviceid), nfsdi_hash) + LIST_FOREACH(dip, &clp->nfsc_devinfo, nfsdi_list) if (NFSBCMP(dip->nfsdi_deviceid, deviceid, NFSX_V4DEVICEID) == 0) break; @@ -4610,10 +4842,11 @@ nfscl_mergeflayouts(struct nfsclflayouth /* * Add this nfscldevinfo to the client, if it doesn't already exist. - * This function consumes the structure pointed at by dip. + * This function consumes the structure pointed at by dip, if not NULL. */ -APPLESTATIC void -nfscl_adddevinfo(struct nfsmount *nmp, struct nfscldevinfo *dip) +APPLESTATIC int +nfscl_adddevinfo(struct nfsmount *nmp, struct nfscldevinfo *dip, + struct nfsclflayout *flp) { struct nfsclclient *clp; struct nfscldevinfo *tdip; @@ -4622,20 +4855,29 @@ nfscl_adddevinfo(struct nfsmount *nmp, s clp = nmp->nm_clp; if (clp == NULL) { NFSUNLOCKCLSTATE(); - free(dip, M_NFSDEVINFO); - return; + if (dip != NULL) + free(dip, M_NFSDEVINFO); + return (ENODEV); } - tdip = nfscl_finddevinfo(clp, dip->nfsdi_deviceid); + tdip = nfscl_finddevinfo(clp, flp->nfsfl_dev); if (tdip != NULL) { + tdip->nfsdi_layoutrefs++; + flp->nfsfl_devp = tdip; nfscl_reldevinfo_locked(tdip); NFSUNLOCKCLSTATE(); - free(dip, M_NFSDEVINFO); - return; + if (dip != NULL) + free(dip, M_NFSDEVINFO); + return (0); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***