Date: Fri, 6 Jul 2018 00:58:52 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r336018 - in projects/pnfs-planb-server/sys/fs: nfs nfsclient Message-ID: <201807060058.w660wqK8099813@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Fri Jul 6 00:58:51 2018 New Revision: 336018 URL: https://svnweb.freebsd.org/changeset/base/336018 Log: Change the pNFS client so that it uses separate TCP connections for same server. This patch changes the pNFS client so that it will use separate TCP connections to the same machine by default. This makes it possible to disable a connection to a DS when multiple DSs reside on the same server. This may be overridden by a sysctl, in case some extant pNFS servers require the same connection to be used for the same server machine. The patch also adds the nfscl_cancelreqs() function to shut down the DS connection if it isn't shared with other DSs. Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h Thu Jul 5 22:56:13 2018 (r336017) +++ projects/pnfs-planb-server/sys/fs/nfs/nfsclstate.h Fri Jul 6 00:58:51 2018 (r336018) @@ -94,6 +94,7 @@ struct nfsclds { #define NFSCLDS_MDS 0x0002 #define NFSCLDS_DS 0x0004 #define NFSCLDS_CLOSED 0x0008 +#define NFSCLDS_SAMECONN 0x0010 struct nfsclclient { LIST_ENTRY(nfsclclient) nfsc_list; Modified: projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c Thu Jul 5 22:56:13 2018 (r336017) +++ projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clrpcops.c Fri Jul 6 00:58:51 2018 (r336018) @@ -57,6 +57,10 @@ static int nfsignore_eexist = 0; SYSCTL_INT(_vfs_nfs, OID_AUTO, ignore_eexist, CTLFLAG_RW, &nfsignore_eexist, 0, "NFS ignore EEXIST replies for mkdir/symlink"); +static int nfscl_dssameconn = 0; +SYSCTL_INT(_vfs_nfs, OID_AUTO, dssameconn, CTLFLAG_RW, + &nfscl_dssameconn, 0, "Use same TCP connection to multiple DSs"); + /* * Global variables */ @@ -162,7 +166,7 @@ static int nfsrpc_writedsmir(vnode_t, int *, int *, nf struct nfsclds *, uint64_t, int, struct nfsfh *, struct mbuf *, int, int, struct ucred *, NFSPROC_T *); static enum nfsclds_state nfscl_getsameserver(struct nfsmount *, - struct nfsclds *, struct nfsclds **); + struct nfsclds *, struct nfsclds **, uint32_t *); static int nfsio_commitds(vnode_t, uint64_t, int, struct nfsclds *, struct nfsfh *, int, int, struct nfsclwritedsdorpc *, struct ucred *, NFSPROC_T *); @@ -5498,9 +5502,14 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in dsp->nfsclds_sockp = nrp; if (vers == NFS_VER4) { NFSLOCKMNT(nmp); - retv = nfscl_getsameserver(nmp, dsp, &tdsp); + retv = nfscl_getsameserver(nmp, dsp, &tdsp, + &sequenceid); NFSCL_DEBUG(3, "getsame ret=%d\n", retv); - if (retv == NFSDSP_USETHISSESSION) { + if (retv == NFSDSP_USETHISSESSION && + nfscl_dssameconn != 0) { + NFSLOCKDS(tdsp); + tdsp->nfsclds_flags |= NFSCLDS_SAMECONN; + NFSUNLOCKDS(tdsp); NFSUNLOCKMNT(nmp); /* * If there is already a session for this @@ -5511,11 +5520,8 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in *dspp = tdsp; return (0); } - if (retv == NFSDSP_SEQTHISSESSION) + if (retv == NFSDSP_NOTFOUND) sequenceid = - tdsp->nfsclds_sess.nfsess_sequenceid; - else - sequenceid = dsp->nfsclds_sess.nfsess_sequenceid; NFSUNLOCKMNT(nmp); error = nfsrpc_createsession(nmp, &dsp->nfsclds_sess, @@ -6518,15 +6524,16 @@ nfscl_freenfsclds(struct nfsclds *dsp) static enum nfsclds_state nfscl_getsameserver(struct nfsmount *nmp, struct nfsclds *newdsp, - struct nfsclds **retdspp) + struct nfsclds **retdspp, uint32_t *sequencep) { - struct nfsclds *dsp, *cur_dsp; + struct nfsclds *dsp; + int fndseq; /* * Search the list of nfsclds structures for one with the same * server. */ - cur_dsp = NULL; + fndseq = 0; TAILQ_FOREACH(dsp, &nmp->nm_sess, nfsclds_list) { if (dsp->nfsclds_servownlen == newdsp->nfsclds_servownlen && dsp->nfsclds_servownlen != 0 && @@ -6536,24 +6543,22 @@ nfscl_getsameserver(struct nfsmount *nmp, struct nfscl NFSCL_DEBUG(4, "fnd same fdsp=%p dsp=%p flg=0x%x\n", TAILQ_FIRST(&nmp->nm_sess), dsp, dsp->nfsclds_flags); + if (fndseq == 0) { + /* Get sequenceid# from first entry. */ + *sequencep = + dsp->nfsclds_sess.nfsess_sequenceid; + fndseq = 1; + } /* Server major id matches. */ if ((dsp->nfsclds_flags & NFSCLDS_DS) != 0) { *retdspp = dsp; return (NFSDSP_USETHISSESSION); } - /* - * Note the first match, so it can be used for - * sequence'ing new sessions. - */ - if (cur_dsp == NULL) - cur_dsp = dsp; } } - if (cur_dsp != NULL) { - *retdspp = cur_dsp; + if (fndseq != 0) return (NFSDSP_SEQTHISSESSION); - } return (NFSDSP_NOTFOUND); } Modified: projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c Thu Jul 5 22:56:13 2018 (r336017) +++ projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clstate.c Fri Jul 6 00:58:51 2018 (r336018) @@ -125,6 +125,7 @@ static struct nfscldeleg *nfscl_finddeleg(struct nfscl static void nfscl_retoncloselayout(vnode_t, struct nfsclclient *, uint8_t *, int, struct nfsclrecalllayout **); static void nfscl_reldevinfo_locked(struct nfscldevinfo *); +static void nfscl_cancelreqs(struct nfsclds *); static struct nfscllayout *nfscl_findlayout(struct nfsclclient *, u_int8_t *, int); static struct nfscldevinfo *nfscl_finddevinfo(struct nfsclclient *, uint8_t *); @@ -4975,11 +4976,12 @@ nfscl_retoncloselayout(vnode_t vp, struct nfsclclient */ void nfscl_dserr(uint32_t op, uint32_t stat, struct nfscldevinfo *dp, - struct nfscllayout *lyp, __unused struct nfsclds *dsp) + struct nfscllayout *lyp, struct nfsclds *dsp) { struct nfsclrecalllayout *recallp; uint32_t iomode; + printf("DS being disabled, error=%d\n", stat); /* Set up the return of the layout. */ recallp = malloc(sizeof(*recallp), M_NFSLAYRECALL, M_WAITOK); iomode = 0; @@ -4998,6 +5000,39 @@ nfscl_dserr(uint32_t op, uint32_t stat, struct nfsclde NFSUNLOCKCLSTATE(); free(recallp, M_NFSLAYRECALL); } + + /* If the connection isn't used for other DSs, we can shut it down. */ + if ((dsp->nfsclds_flags & NFSCLDS_SAMECONN) == 0) + nfscl_cancelreqs(dsp); +} + +/* + * Cancel all RPCs for this "dsp" by closing the connection. + * Also, mark the session as defunct. + */ +static void +nfscl_cancelreqs(struct nfsclds *dsp) +{ + struct __rpc_client *cl; + static int non_event; + + NFSLOCKDS(dsp); + if ((dsp->nfsclds_flags & (NFSCLDS_CLOSED | NFSCLDS_SAMECONN)) == 0 && + dsp->nfsclds_sockp != NULL && + dsp->nfsclds_sockp->nr_client != NULL) { + dsp->nfsclds_flags |= NFSCLDS_CLOSED; + cl = dsp->nfsclds_sockp->nr_client; + dsp->nfsclds_sess.nfsess_defunct = 1; + NFSUNLOCKDS(dsp); + CLNT_CLOSE(cl); + /* + * This 1sec sleep is done to give the MDS time to recall + * layouts before this client attempts further I/O. + */ + tsleep(&non_event, PVFS, "ndscls", hz); + return; + } + NFSUNLOCKDS(dsp); } /*
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201807060058.w660wqK8099813>