Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 14 Jul 2018 19:51:44 +0000 (UTC)
From:      Rick Macklem <rmacklem@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r336292 - in head/sys/fs: nfs nfsclient
Message-ID:  <201807141951.w6EJpicC026731@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: rmacklem
Date: Sat Jul 14 19:51:44 2018
New Revision: 336292
URL: https://svnweb.freebsd.org/changeset/base/336292

Log:
  Fix the pNFS client when mirrors aren't on the same machine.
  
  Without this patch, the client side NFSv4.1 pNFS code erroneously did writes
  and commits to both DS mirrors using the TCP connection of the first one.
  For my test setup this worked, since I have both DSs running on the same
  machine, but it would have failed when the DSs are on separate machines.
  This patch fixes the code to use the correct TCP connection for each DS.
  This patch should only affect the NFSv4.1 client when using "pnfs" mounts
  to mirrored DSs.
  
  MFC after:	2 weeks

Modified:
  head/sys/fs/nfs/nfs_var.h
  head/sys/fs/nfs/nfsclstate.h
  head/sys/fs/nfsclient/nfs_clrpcops.c
  head/sys/fs/nfsclient/nfs_clstate.c

Modified: head/sys/fs/nfs/nfs_var.h
==============================================================================
--- head/sys/fs/nfs/nfs_var.h	Sat Jul 14 19:49:14 2018	(r336291)
+++ head/sys/fs/nfs/nfs_var.h	Sat Jul 14 19:51:44 2018	(r336292)
@@ -607,7 +607,7 @@ void nfscl_rellayout(struct nfscllayout *, int);
 struct nfscldevinfo *nfscl_getdevinfo(struct nfsclclient *, uint8_t *,
     struct nfscldevinfo *);
 void nfscl_reldevinfo(struct nfscldevinfo *);
-int nfscl_adddevinfo(struct nfsmount *, struct nfscldevinfo *,
+int nfscl_adddevinfo(struct nfsmount *, struct nfscldevinfo *, int,
     struct nfsclflayout *);
 void nfscl_freelayout(struct nfscllayout *);
 void nfscl_freeflayout(struct nfsclflayout *);

Modified: head/sys/fs/nfs/nfsclstate.h
==============================================================================
--- head/sys/fs/nfs/nfsclstate.h	Sat Jul 14 19:49:14 2018	(r336291)
+++ head/sys/fs/nfs/nfsclstate.h	Sat Jul 14 19:51:44 2018	(r336292)
@@ -269,6 +269,7 @@ struct nfscllayout {
  */
 struct nfsffm {
 	nfsv4stateid_t		st;
+	struct nfscldevinfo	*devp;
 	char			dev[NFSX_V4DEVICEID];
 	uint32_t		eff;
 	uid_t			user;
@@ -290,7 +291,6 @@ struct nfsclflayout {
 	uint64_t			nfsfl_off;
 	uint64_t			nfsfl_end;
 	uint32_t			nfsfl_iomode;
-	struct nfscldevinfo		*nfsfl_devp;
 	uint16_t			nfsfl_flags;
 	union {
 		struct {
@@ -299,6 +299,7 @@ struct nfsclflayout {
 			uint32_t	stripe1;
 			uint8_t		dev[NFSX_V4DEVICEID];
 			uint16_t	fhcnt;
+			struct nfscldevinfo *devp;
 		} fl;
 		struct {
 			uint64_t	stripeunit;
@@ -317,6 +318,7 @@ struct nfsclflayout {
 #define	nfsfl_stripe1		nfsfl_un.fl.stripe1
 #define	nfsfl_dev		nfsfl_un.fl.dev
 #define	nfsfl_fhcnt		nfsfl_un.fl.fhcnt
+#define	nfsfl_devp		nfsfl_un.fl.devp
 #define	nfsfl_stripeunit	nfsfl_un.ff.stripeunit
 #define	nfsfl_fflags		nfsfl_un.ff.fflags
 #define	nfsfl_statshint		nfsfl_un.ff.statshint

Modified: head/sys/fs/nfsclient/nfs_clrpcops.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clrpcops.c	Sat Jul 14 19:49:14 2018	(r336291)
+++ head/sys/fs/nfsclient/nfs_clrpcops.c	Sat Jul 14 19:51:44 2018	(r336292)
@@ -5722,12 +5722,15 @@ nfscl_doiods(vnode_t vp, struct uio *uiop, int *iomode
 				}
 			}
 			for (i = firstmirror; i < mirrorcnt && error == 0; i++){
-				if ((layp->nfsly_flags & NFSLY_FLEXFILE) != 0)
+				if ((layp->nfsly_flags & NFSLY_FLEXFILE) != 0) {
 					dev = rflp->nfsfl_ffm[i].dev;
-				else
+					dip = nfscl_getdevinfo(nmp->nm_clp, dev,
+					    rflp->nfsfl_ffm[i].devp);
+				} else {
 					dev = rflp->nfsfl_dev;
-				dip = nfscl_getdevinfo(nmp->nm_clp, dev,
-				    rflp->nfsfl_devp);
+					dip = nfscl_getdevinfo(nmp->nm_clp, dev,
+					    rflp->nfsfl_devp);
+				}
 				if (dip != NULL) {
 					if ((rflp->nfsfl_flags & NFSFL_FLEXFILE)
 					    != 0)
@@ -6828,7 +6831,8 @@ nfsrv_parselayoutget(struct nfsrv_descript *nd, nfsv4s
 			    sizeof(struct nfsffm), M_NFSFLAYOUT, M_WAITOK);
 			flp->nfsfl_flags = NFSFL_FLEXFILE;
 			flp->nfsfl_mirrorcnt = mirrorcnt;
-			flp->nfsfl_devp = NULL;
+			for (j = 0; j < mirrorcnt; j++)
+				flp->nfsfl_ffm[j].devp = NULL;
 			flp->nfsfl_off = off;
 			if (flp->nfsfl_off + retlen < flp->nfsfl_off)
 				flp->nfsfl_end = UINT64_MAX - flp->nfsfl_off;
@@ -7588,6 +7592,7 @@ nfsrpc_layoutgetres(struct nfsmount *nmp, vnode_t vp, 
 	struct nfsclflayout *tflp;
 	struct nfscldevinfo *dip;
 	uint8_t *dev;
+	int i, mirrorcnt;
 
 	if (laystat == NFSERR_UNKNLAYOUTTYPE) {
 		NFSLOCKMNT(nmp);
@@ -7604,25 +7609,35 @@ nfsrpc_layoutgetres(struct nfsmount *nmp, vnode_t vp, 
 	if (laystat == 0) {
 		NFSCL_DEBUG(4, "nfsrpc_layoutgetres at FOREACH\n");
 		LIST_FOREACH(tflp, flhp, nfsfl_list) {
-			laystat = nfscl_adddevinfo(nmp, NULL, tflp);
-			NFSCL_DEBUG(4, "aft adddev=%d\n", laystat);
-			if (laystat != 0) {
-				if (layouttype == NFSLAYOUT_FLEXFILE)
-					dev = tflp->nfsfl_ffm[0].dev;
-				else
-					dev = tflp->nfsfl_dev;
-				laystat = nfsrpc_getdeviceinfo(nmp, dev,
-				    layouttype, notifybit, &dip, cred, p);
-				NFSCL_DEBUG(4, "aft nfsrpc_gdi=%d\n",
-				    laystat);
-				if (laystat != 0)
-					break;
-				laystat = nfscl_adddevinfo(nmp, dip, tflp);
-				if (laystat != 0)
-					printf("getlayout: cannot add\n");
+			if (layouttype == NFSLAYOUT_FLEXFILE)
+				mirrorcnt = tflp->nfsfl_mirrorcnt;
+			else
+				mirrorcnt = 1;
+			for (i = 0; i < mirrorcnt; i++) {
+				laystat = nfscl_adddevinfo(nmp, NULL, i, tflp);
+				NFSCL_DEBUG(4, "aft adddev=%d\n", laystat);
+				if (laystat != 0) {
+					if (layouttype == NFSLAYOUT_FLEXFILE)
+						dev = tflp->nfsfl_ffm[i].dev;
+					else
+						dev = tflp->nfsfl_dev;
+					laystat = nfsrpc_getdeviceinfo(nmp, dev,
+					    layouttype, notifybit, &dip, cred,
+					    p);
+					NFSCL_DEBUG(4, "aft nfsrpc_gdi=%d\n",
+					    laystat);
+					if (laystat != 0)
+						goto out;
+					laystat = nfscl_adddevinfo(nmp, dip, i,
+					    tflp);
+					if (laystat != 0)
+						printf("nfsrpc_layoutgetresout"
+						    ": cannot add\n");
+				}
 			}
 		}
 	}
+out:
 	if (laystat == 0) {
 		/*
 		 * nfscl_layout() always returns with the nfsly_lock

Modified: head/sys/fs/nfsclient/nfs_clstate.c
==============================================================================
--- head/sys/fs/nfsclient/nfs_clstate.c	Sat Jul 14 19:49:14 2018	(r336291)
+++ head/sys/fs/nfsclient/nfs_clstate.c	Sat Jul 14 19:51:44 2018	(r336292)
@@ -5152,7 +5152,7 @@ nfscl_mergeflayouts(struct nfsclflayouthead *fhlp,
  * This function consumes the structure pointed at by dip, if not NULL.
  */
 APPLESTATIC int
-nfscl_adddevinfo(struct nfsmount *nmp, struct nfscldevinfo *dip,
+nfscl_adddevinfo(struct nfsmount *nmp, struct nfscldevinfo *dip, int ind,
     struct nfsclflayout *flp)
 {
 	struct nfsclclient *clp;
@@ -5170,11 +5170,14 @@ nfscl_adddevinfo(struct nfsmount *nmp, struct nfscldev
 	if ((flp->nfsfl_flags & NFSFL_FILE) != 0)
 		dev = flp->nfsfl_dev;
 	else
-		dev = flp->nfsfl_ffm[0].dev;
+		dev = flp->nfsfl_ffm[ind].dev;
 	tdip = nfscl_finddevinfo(clp, dev);
 	if (tdip != NULL) {
 		tdip->nfsdi_layoutrefs++;
-		flp->nfsfl_devp = tdip;
+		if ((flp->nfsfl_flags & NFSFL_FILE) != 0)
+			flp->nfsfl_devp = tdip;
+		else
+			flp->nfsfl_ffm[ind].devp = tdip;
 		nfscl_reldevinfo_locked(tdip);
 		NFSUNLOCKCLSTATE();
 		if (dip != NULL)
@@ -5184,7 +5187,10 @@ nfscl_adddevinfo(struct nfsmount *nmp, struct nfscldev
 	if (dip != NULL) {
 		LIST_INSERT_HEAD(&clp->nfsc_devinfo, dip, nfsdi_list);
 		dip->nfsdi_layoutrefs = 1;
-		flp->nfsfl_devp = dip;
+		if ((flp->nfsfl_flags & NFSFL_FILE) != 0)
+			flp->nfsfl_devp = dip;
+		else
+			flp->nfsfl_ffm[ind].devp = dip;
 	}
 	NFSUNLOCKCLSTATE();
 	if (dip == NULL)
@@ -5225,15 +5231,19 @@ nfscl_freeflayout(struct nfsclflayout *flp)
 {
 	int i, j;
 
-	if ((flp->nfsfl_flags & NFSFL_FILE) != 0)
+	if ((flp->nfsfl_flags & NFSFL_FILE) != 0) {
 		for (i = 0; i < flp->nfsfl_fhcnt; i++)
 			free(flp->nfsfl_fh[i], M_NFSFH);
+		if (flp->nfsfl_devp != NULL)
+			flp->nfsfl_devp->nfsdi_layoutrefs--;
+	}
 	if ((flp->nfsfl_flags & NFSFL_FLEXFILE) != 0)
-		for (i = 0; i < flp->nfsfl_mirrorcnt; i++)
+		for (i = 0; i < flp->nfsfl_mirrorcnt; i++) {
 			for (j = 0; j < flp->nfsfl_ffm[i].fhcnt; j++)
 				free(flp->nfsfl_ffm[i].fh[j], M_NFSFH);
-	if (flp->nfsfl_devp != NULL)
-		flp->nfsfl_devp->nfsdi_layoutrefs--;
+			if (flp->nfsfl_ffm[i].devp != NULL)	
+				flp->nfsfl_ffm[i].devp->nfsdi_layoutrefs--;	
+		}
 	free(flp, M_NFSFLAYOUT);
 }
 



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201807141951.w6EJpicC026731>