Date: Tue, 29 May 2018 03:19:56 +0000 (UTC) From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r334309 - in projects/pnfs-planb-server/sys/fs: nfs nfsclient nfsserver Message-ID: <201805290319.w4T3JuNC023690@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: rmacklem Date: Tue May 29 03:19:56 2018 New Revision: 334309 URL: https://svnweb.freebsd.org/changeset/base/334309 Log: Change the pNFS server so that it does a distributed layout of mirrored storage files on the DSs instead of strict sets of mirrored DSs. This actually simplified the code in various places and made doing migration easy. It also implies that typically only one DS will run out of space at a time and can be disabled, allowing the service to continue to run until other DSs run out of space. It also adds the capability of migration, where a sysadmin can move data storage files between the DSs. The system call that does this could be used by a load/storage space balancer in the future. Patched versions of nfsd and pnfsdscopymr will be committed soon to make this work. Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs.h projects/pnfs-planb-server/sys/fs/nfs/nfs_commonsubs.c projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clport.c projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clvfsops.c projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs.h ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfs.h Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfs/nfs.h Tue May 29 03:19:56 2018 (r334309) @@ -185,18 +185,14 @@ struct nfsd_nfsd_args { int dnshostlen; /* Length of DNS names */ char *dspath; /* DS Mount path on MDS */ int dspathlen; /* Length of DS Mount path on MDS */ - char *mirror; /* DS with same mirrorindex are mirrors */ - int mirrorlen; /* Length of the mirrorindex array */ + int mirrorcnt; /* Number of mirrors to create on DSs */ }; /* - * NFSDEV_MIRRORSTR - string of digits that number the DSs 0->999. - * (To support more than 1000 DSs on an MDS, this needs to be increased.) * NFSDEV_MAXMIRRORS - Maximum # of mirrors for a DS. * (Most will only have a single mirror, but this setting allows up to 3.) * NFSDEV_MAXVERS - maximum number of NFS versions supported by Flex File. */ -#define NFSDEV_MIRRORSTR 3 #define NFSDEV_MAXMIRRORS 4 #define NFSDEV_MAXVERS 4 Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_commonsubs.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfs_commonsubs.c Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfs/nfs_commonsubs.c Tue May 29 03:19:56 2018 (r334309) @@ -71,7 +71,6 @@ int nfsrv_lease = NFSRV_LEASE; int ncl_mbuf_mlen = MLEN; int nfsd_enable_stringtouid = 0; int nfsrv_doflexfile = 0; -int nfsrv_maxpnfsmirror = 1; static int nfs_enable_uidtostring = 0; NFSNAMEIDMUTEX; NFSSOCKMUTEX; @@ -85,6 +84,10 @@ SYSCTL_DECL(_vfs_nfs); SYSCTL_INT(_vfs_nfs, OID_AUTO, enable_uidtostring, CTLFLAG_RW, &nfs_enable_uidtostring, 0, "Make nfs always send numeric owner_names"); +int nfsrv_maxpnfsmirror = 1; +SYSCTL_INT(_vfs_nfs, OID_AUTO, pnfsmirror, CTLFLAG_RD, + &nfsrv_maxpnfsmirror, 0, "Mirror level for pNFS service"); + /* * This array of structures indicates, for V4: * retfh - which of 3 types of calling args are used @@ -4313,47 +4316,32 @@ nfsv4_freeslot(struct nfsclsession *sep, int slot) * Return one if found, NULL otherwise. */ struct nfsdevice * -nfsv4_findmirror(struct nfsmount *nmp, struct nfsdevice **fndpardsp) +nfsv4_findmirror(struct nfsmount *nmp) { - struct nfsdevice *ds, *mds, *fndds; + struct nfsdevice *ds, *fndds; + int fndmirror; mtx_assert(NFSDDSMUTEXPTR, MA_OWNED); /* - * Search the DS server list for a match with dvp. + * Search the DS server list for a match with nmp. * Remove the DS entry if found and there is a mirror. */ fndds = NULL; - if (fndpardsp != NULL) - *fndpardsp = NULL; + fndmirror = 0; if (nfsrv_devidcnt == 0) return (fndds); TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) { - if (fndds != NULL) - break; if (ds->nfsdev_nmp == nmp) { - /* If there are no mirrors, return NULL. */ - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, nfsdev_list) { - if (mds->nfsdev_nmp != NULL) - break; - } - if (mds == NULL) { - NFSCL_DEBUG(4, "no mirror for DS\n"); - return (NULL); - } NFSCL_DEBUG(4, "fnd main ds\n"); fndds = ds; + } else if (ds->nfsdev_nmp != NULL) + fndmirror = 1; + if (fndds != NULL && fndmirror != 0) break; - } else { - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, nfsdev_list) { - if (mds->nfsdev_nmp == nmp) { - NFSCL_DEBUG(4, "fnd mirror ds\n"); - fndds = mds; - if (fndpardsp != NULL) - *fndpardsp = ds; - break; - } - } - } + } + if (fndmirror == 0) { + NFSCL_DEBUG(4, "no mirror for DS\n"); + return (NULL); } return (fndds); } Modified: projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfs/nfs_var.h Tue May 29 03:19:56 2018 (r334309) @@ -162,10 +162,9 @@ struct nfsdevice *nfsrv_deldsnmp(struct nfsmount *, NF int nfsrv_createdevids(struct nfsd_nfsd_args *, NFSPROC_T *); int nfsrv_checkdsattr(struct nfsrv_descript *, vnode_t, NFSPROC_T *); int nfsrv_copymr(vnode_t, vnode_t, vnode_t, struct nfsdevice *, - struct pnfsdsfile *, struct pnfsdsfile *, int, struct ucred *, - NFSPROC_T *); -int nfsrv_mdscopymr(char *, char *, char *, char *, int *, NFSPROC_T *, - struct vnode **, struct vnode **, struct pnfsdsfile **, + struct pnfsdsfile *, struct pnfsdsfile *, int, struct ucred *, NFSPROC_T *); +int nfsrv_mdscopymr(char *, char *, char *, char *, int *, char *, NFSPROC_T *, + struct vnode **, struct vnode **, struct pnfsdsfile **, struct nfsdevice **, struct nfsdevice **); /* nfs_nfsdserv.c */ @@ -339,7 +338,7 @@ int nfsv4_sequencelookup(struct nfsmount *, struct nfs int *, uint32_t *, uint8_t *); void nfsv4_freeslot(struct nfsclsession *, int); struct ucred *nfsrv_getgrpscred(struct ucred *); -struct nfsdevice *nfsv4_findmirror(struct nfsmount *, struct nfsdevice **); +struct nfsdevice *nfsv4_findmirror(struct nfsmount *); /* nfs_clcomsubs.c */ void nfsm_uiombuf(struct nfsrv_descript *, struct uio *, int); @@ -706,7 +705,7 @@ int nfsrv_dsgetdevandfh(struct vnode *, NFSPROC_T *, i char *); int nfsrv_dsgetsockmnt(struct vnode *, int, char *, int *, int *, NFSPROC_T *, struct vnode **, fhandle_t *, char *, char *, - struct vnode **, struct nfsmount *, struct nfsmount *, int *, int *); + struct vnode **, struct nfsmount **, struct nfsmount *, int *, int *); int nfsrv_dscreate(struct vnode *, struct vattr *, struct vattr *, fhandle_t *, struct pnfsdsfile *, struct pnfsdsattr *, char *, struct ucred *, NFSPROC_T *, struct vnode **); Modified: projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfs/nfsrvstate.h Tue May 29 03:19:56 2018 (r334309) @@ -343,11 +343,9 @@ void nfsrv_freedeleglist(struct nfsstatehead *); */ struct nfsdevice { TAILQ_ENTRY(nfsdevice) nfsdev_list; - struct nfsdevicehead nfsdev_mirrors; vnode_t nfsdev_dvp; struct nfsmount *nfsdev_nmp; char nfsdev_deviceid[NFSX_V4DEVICEID]; - char nfsdev_mirrorid[NFSDEV_MIRRORSTR + 1]; uint16_t nfsdev_hostnamelen; uint16_t nfsdev_fileaddrlen; uint16_t nfsdev_flexaddrlen; Modified: projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clport.c Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clport.c Tue May 29 03:19:56 2018 (r334309) @@ -1385,8 +1385,7 @@ nfssvc_nfscl(struct thread *td, struct nfssvc_args *ua "nfs") == 0 && mp->mnt_data != NULL) { nmp = VFSTONFS(mp); NFSDDSLOCK(); - if (nfsv4_findmirror(nmp, NULL) != NULL - ) { + if (nfsv4_findmirror(nmp) != NULL) { NFSDDSUNLOCK(); error = ENXIO; nmp = NULL; Modified: projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clvfsops.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clvfsops.c Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfsclient/nfs_clvfsops.c Tue May 29 03:19:56 2018 (r334309) @@ -1683,7 +1683,7 @@ nfs_unmount(struct mount *mp, int mntflags) /* In the forced case, cancel any outstanding requests. */ if (mntflags & MNT_FORCE) { NFSDDSLOCK(); - if (nfsv4_findmirror(nmp, NULL) != NULL) + if (nfsv4_findmirror(nmp) != NULL) error = ENXIO; NFSDDSUNLOCK(); if (error) Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Tue May 29 03:19:56 2018 (r334309) @@ -73,6 +73,7 @@ extern int nfs_pnfsiothreads; extern struct nfsdontlisthead nfsrv_dontlisthead; extern volatile int nfsrv_dontlistlen; extern volatile int nfsrv_devidcnt; +extern int nfsrv_maxpnfsmirror; struct vfsoptlist nfsv4root_opt, nfsv4root_newopt; NFSDLOCKMUTEX; NFSSTATESPINLOCK; @@ -129,6 +130,7 @@ static int nfsrv_pnfslookupds(struct vnode *, struct v struct pnfsdsfile *, struct vnode **, NFSPROC_T *); static void nfsrv_pnfssetfh(struct vnode *, struct pnfsdsfile *, struct vnode *, NFSPROC_T *); +static int nfsrv_dsremove(struct vnode *, char *, struct ucred *, NFSPROC_T *); int nfs_pnfsio(task_fn_t *, void *); @@ -3282,12 +3284,13 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap struct nfsd_nfsd_args nfsdarg; struct nfsd_nfsd_oargs onfsdarg; struct nfsd_pnfsd_args pnfsdarg; - struct vnode *vp, *nvp; + struct vnode *vp, *nvp, *curdvp; struct pnfsdsfile *pf; - struct nfsdevice *ds; + struct nfsdevice *ds, *fds; cap_rights_t rights; - int buflen, error; + int buflen, error, ret; char *buf, *cp, *cp2, *cp3; + char fname[PNFS_FILENAME_LEN + 1]; if (uap->flag & NFSSVC_NFSDADDSOCK) { error = copyin(uap->argp, (caddr_t)&sockarg, sizeof (sockarg)); @@ -3325,8 +3328,7 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap nfsdarg.addrlen = 0; nfsdarg.dnshost = NULL; nfsdarg.dnshostlen = 0; - nfsdarg.mirror = NULL; - nfsdarg.mirrorlen = 0; + nfsdarg.mirrorcnt = 1; } } else error = copyin(uap->argp, &nfsdarg, sizeof(nfsdarg)); @@ -3335,13 +3337,14 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap if (nfsdarg.addrlen > 0 && nfsdarg.addrlen < 10000 && nfsdarg.dnshostlen > 0 && nfsdarg.dnshostlen < 10000 && nfsdarg.dspathlen > 0 && nfsdarg.dspathlen < 10000 && - nfsdarg.mirrorlen > 0 && nfsdarg.mirrorlen < 10000 && + nfsdarg.mirrorcnt >= 1 && + nfsdarg.mirrorcnt <= NFSDEV_MAXMIRRORS && nfsdarg.addr != NULL && nfsdarg.dnshost != NULL && - nfsdarg.dspath != NULL && nfsdarg.mirror != NULL) { + nfsdarg.dspath != NULL) { NFSD_DEBUG(1, "addrlen=%d dspathlen=%d dnslen=%d" - " mirrorlen=%d\n", nfsdarg.addrlen, + " mirrorcnt=%d\n", nfsdarg.addrlen, nfsdarg.dspathlen, nfsdarg.dnshostlen, - nfsdarg.mirrorlen); + nfsdarg.mirrorcnt); cp = malloc(nfsdarg.addrlen + 1, M_TEMP, M_WAITOK); error = copyin(nfsdarg.addr, cp, nfsdarg.addrlen); if (error != 0) { @@ -3369,17 +3372,6 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap } cp[nfsdarg.dspathlen] = '\0'; /* Ensure nul term. */ nfsdarg.dspath = cp; - cp = malloc(nfsdarg.mirrorlen + 1, M_TEMP, M_WAITOK); - error = copyin(nfsdarg.mirror, cp, nfsdarg.mirrorlen); - if (error != 0) { - free(nfsdarg.addr, M_TEMP); - free(nfsdarg.dnshost, M_TEMP); - free(nfsdarg.dspath, M_TEMP); - free(cp, M_TEMP); - goto out; - } - cp[nfsdarg.mirrorlen] = '\0'; /* Ensure nul term. */ - nfsdarg.mirror = cp; } else { nfsdarg.addr = NULL; nfsdarg.addrlen = 0; @@ -3387,14 +3379,12 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap nfsdarg.dnshostlen = 0; nfsdarg.dspath = NULL; nfsdarg.dspathlen = 0; - nfsdarg.mirror = NULL; - nfsdarg.mirrorlen = 0; + nfsdarg.mirrorcnt = 1; } error = nfsrvd_nfsd(td, &nfsdarg); free(nfsdarg.addr, M_TEMP); free(nfsdarg.dnshost, M_TEMP); free(nfsdarg.dspath, M_TEMP); - free(nfsdarg.mirror, M_TEMP); } else if (uap->flag & NFSSVC_PNFSDS) { error = copyin(uap->argp, &pnfsdarg, sizeof(pnfsdarg)); if (error == 0 && pnfsdarg.op == PNFSDOP_DELDSSERVER) { @@ -3406,25 +3396,33 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap free(cp, M_TEMP); } else if (error == 0 && pnfsdarg.op == PNFSDOP_COPYMR) { cp = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK); - cp2 = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK); - cp3 = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK); buflen = sizeof(*pf) * NFSDEV_MAXMIRRORS; buf = malloc(buflen, M_TEMP, M_WAITOK); error = copyinstr(pnfsdarg.mdspath, cp, PATH_MAX + 1, NULL); - NFSD_DEBUG(4, "pnfsdop_copymr cp mdspath=%d\n", error); - if (error == 0) + NFSD_DEBUG(4, "pnfsdcopymr cp mdspath=%d\n", error); + if (error == 0 && pnfsdarg.dspath != NULL) { + cp2 = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK); error = copyinstr(pnfsdarg.dspath, cp2, PATH_MAX + 1, NULL); - NFSD_DEBUG(4, "pnfsdop_copymr cp dspath=%d\n", error); - if (error == 0) + NFSD_DEBUG(4, "pnfsdcopymr cp dspath=%d\n", + error); + } else + cp2 = NULL; + if (error == 0 && pnfsdarg.curdspath != NULL) { + cp3 = malloc(PATH_MAX + 1, M_TEMP, M_WAITOK); error = copyinstr(pnfsdarg.curdspath, cp3, PATH_MAX + 1, NULL); - NFSD_DEBUG(4, "pnfsdop_copymr cp curdspath=%d\n", - error); + NFSD_DEBUG(4, "pnfsdcopymr cp curdspath=%d\n", + error); + } else + cp3 = NULL; + curdvp = NULL; + fds = NULL; if (error == 0) error = nfsrv_mdscopymr(cp, cp2, cp3, buf, - &buflen, td, &vp, &nvp, &pf, &ds); + &buflen, fname, td, &vp, &nvp, &pf, &ds, + &fds); NFSD_DEBUG(4, "nfsrv_mdscopymr=%d\n", error); if (error == 0) { if (pf->dsf_dir >= nfsrv_dsdirsize) { @@ -3438,6 +3436,15 @@ nfssvc_nfsd(struct thread *td, struct nfssvc_args *uap buflen / sizeof(*pf), td->td_ucred, td); vput(vp); vput(nvp); + if (fds != NULL && error == 0) { + curdvp = fds->nfsdev_dsdir[pf->dsf_dir]; + ret = vn_lock(curdvp, LK_SHARED); + if (ret == 0) { + nfsrv_dsremove(curdvp, fname, + td->td_ucred, td); + NFSVOPUNLOCK(curdvp, 0); + } + } NFSD_DEBUG(4, "nfsrv_copymr=%d\n", error); } free(cp, M_TEMP); @@ -3813,25 +3820,33 @@ nfsrv_pnfscreate(struct vnode *vp, struct vattr *vap, /* Get a DS server directory in a round-robin order. */ mirrorcnt = 1; NFSDDSLOCK(); - ds = TAILQ_FIRST(&nfsrv_devidhead); + TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) { + if (ds->nfsdev_nmp != NULL) + break; + } if (ds == NULL) { NFSDDSUNLOCK(); NFSD_DEBUG(4, "nfsrv_pnfscreate: no srv\n"); return; } - /* Put at end of list to implement round-robin usage. */ - TAILQ_REMOVE(&nfsrv_devidhead, ds, nfsdev_list); - TAILQ_INSERT_TAIL(&nfsrv_devidhead, ds, nfsdev_list); i = dsdir[0] = ds->nfsdev_nextdir; ds->nfsdev_nextdir = (ds->nfsdev_nextdir + 1) % nfsrv_dsdirsize; dvp[0] = ds->nfsdev_dsdir[i]; - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, nfsdev_list) { - if (mds->nfsdev_nmp != NULL) { - dsdir[mirrorcnt] = i; - dvp[mirrorcnt] = mds->nfsdev_dsdir[i]; - mirrorcnt++; + if (nfsrv_maxpnfsmirror > 1) { + mds = TAILQ_NEXT(ds, nfsdev_list); + TAILQ_FOREACH_FROM(mds, &nfsrv_devidhead, nfsdev_list) { + if (mds->nfsdev_nmp != NULL) { + dsdir[mirrorcnt] = i; + dvp[mirrorcnt] = mds->nfsdev_dsdir[i]; + mirrorcnt++; + if (mirrorcnt >= nfsrv_maxpnfsmirror) + break; + } } } + /* Put at end of list to implement round-robin usage. */ + TAILQ_REMOVE(&nfsrv_devidhead, ds, nfsdev_list); + TAILQ_INSERT_TAIL(&nfsrv_devidhead, ds, nfsdev_list); NFSDDSUNLOCK(); dsc = NULL; if (mirrorcnt > 1) @@ -4391,21 +4406,25 @@ tryagain: /* * Get the DS mount point, fh and directory from the "pnfsd.dsfile" extended * attribute. + * newnmpp - If it points to a non-NULL nmp, that is the destination and needs + * to be checked. If it points to a NULL nmp, then it returns + * a suitable destination. + * curnmp - If non-NULL, it is the source mount for the copy. */ int nfsrv_dsgetsockmnt(struct vnode *vp, int lktype, char *buf, int *buflenp, int *mirrorcntp, NFSPROC_T *p, struct vnode **dvpp, fhandle_t *fhp, - char *devid, char *fnamep, struct vnode **nvpp, struct nfsmount *newnmp, - struct nfsmount *curnmp, int *zeroippos, int *dsdirp) + char *devid, char *fnamep, struct vnode **nvpp, struct nfsmount **newnmpp, + struct nfsmount *curnmp, int *ippos, int *dsdirp) { struct vnode *dvp, *nvp, **tdvpp; - struct nfsmount *nmp; + struct nfsmount *nmp, *newnmp; struct sockaddr *sad; struct sockaddr_in *sin; - struct nfsdevice *ds, *mds; + struct nfsdevice *ds, *fndds; struct pnfsdsfile *pf; uint32_t dsdir; - int done, error, fnd, fhiszero, gotone, i, mirrorcnt; + int error, fhiszero, fnd, gotone, i, mirrorcnt; ASSERT_VOP_LOCKED(vp, "nfsrv_dsgetsockmnt vp"); *mirrorcntp = 1; @@ -4416,8 +4435,12 @@ nfsrv_dsgetsockmnt(struct vnode *vp, int lktype, char *nvpp = NULL; if (dvpp != NULL) *dvpp = NULL; - if (zeroippos != NULL) - *zeroippos = -1; + if (ippos != NULL) + *ippos = -1; + if (newnmpp != NULL) + newnmp = *newnmpp; + else + newnmp = NULL; error = vn_extattr_get(vp, IO_NODELOCKED, EXTATTR_NAMESPACE_SYSTEM, "pnfsd.dsfile", buflenp, buf, p); mirrorcnt = *buflenp / sizeof(*pf); @@ -4432,13 +4455,14 @@ nfsrv_dsgetsockmnt(struct vnode *vp, int lktype, char for (i = 0; i < mirrorcnt; i++, pf++) { sad = (struct sockaddr *)&pf->dsf_sin; if (nfsaddr2_match(sad, curnmp->nm_nam)) { + if (ippos != NULL) + *ippos = i; fnd = 1; break; } } - /* Set EEXIST so that pnfsdscopymr won't report an error. */ if (fnd == 0) - error = EEXIST; + error = ENXIO; } gotone = 0; @@ -4453,50 +4477,46 @@ nfsrv_dsgetsockmnt(struct vnode *vp, int lktype, char if (dsdir >= nfsrv_dsdirsize) { printf("nfsrv_dsgetsockmnt: dsdir=%d\n", dsdir); error = ENOATTR; - } else if (nvpp != NULL && nfsaddr2_match(sad, newnmp->nm_nam)) + } else if (nvpp != NULL && newnmp != NULL && + nfsaddr2_match(sad, newnmp->nm_nam)) error = EEXIST; if (error == 0) { - if (zeroippos != NULL && sad->sa_family == AF_INET && + if (ippos != NULL && curnmp == NULL && + sad->sa_family == AF_INET && sin->sin_addr.s_addr == 0) - *zeroippos = i; + *ippos = i; if (NFSBCMP(&zerofh, &pf->dsf_fh, sizeof(zerofh)) == 0) fhiszero = 1; /* Use the socket address to find the mount point. */ - done = 0; + fndds = NULL; NFSDDSLOCK(); TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) { - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, - nfsdev_list) { - if (mds->nfsdev_nmp != NULL) { - dvp = mds->nfsdev_dvp; - nmp = VFSTONFS(dvp->v_mount); - if (nmp != mds->nfsdev_nmp) - printf("different nmp " - "%p %p\n", nmp, - mds->nfsdev_nmp); - if (nfsaddr2_match(sad, - nmp->nm_nam)) { - ds = mds; - done = 1; - break; - } - } + if (ds->nfsdev_nmp != NULL) { + dvp = ds->nfsdev_dvp; + nmp = VFSTONFS(dvp->v_mount); + if (nmp != ds->nfsdev_nmp) + printf("different2 nmp %p %p\n", + nmp, ds->nfsdev_nmp); + if (nfsaddr2_match(sad, nmp->nm_nam)) + fndds = ds; + else if (newnmpp != NULL && + newnmp == NULL && + (*newnmpp == NULL || fndds == NULL)) + /* + * Return a destination for the + * copy in newnmpp. Choose the + * last valid one before the + * source mirror, so it isn't + * always the first one. + */ + *newnmpp = nmp; } - if (done != 0) - break; - dvp = ds->nfsdev_dvp; - nmp = VFSTONFS(dvp->v_mount); - if (nmp != ds->nfsdev_nmp) - printf("different2 nmp %p %p\n", - nmp, ds->nfsdev_nmp); - if (nfsaddr2_match(sad, nmp->nm_nam)) - break; } NFSDDSUNLOCK(); - if (ds != NULL) { + if (fndds != NULL) { if (dvpp != NULL || fhiszero != 0 || (nvpp != NULL && *nvpp == NULL)) { - dvp = ds->nfsdev_dsdir[dsdir]; + dvp = fndds->nfsdev_dsdir[dsdir]; error = vn_lock(dvp, lktype); /* * If the file handle is all 0's, try to @@ -4529,7 +4549,7 @@ nfsrv_dsgetsockmnt(struct vnode *vp, int lktype, char gotone++; NFSD_DEBUG(4, "gotone=%d\n", gotone); if (devid != NULL) { - NFSBCOPY(ds->nfsdev_deviceid, + NFSBCOPY(fndds->nfsdev_deviceid, devid, NFSX_V4DEVICEID); devid += NFSX_V4DEVICEID; } Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c ============================================================================== --- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c Tue May 29 02:41:32 2018 (r334308) +++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdstate.c Tue May 29 03:19:56 2018 (r334309) @@ -208,10 +208,10 @@ static void nfsrv_freelayouts(nfsquad_t *clid, fsid_t int iomode); static void nfsrv_freealllayouts(void); static void nfsrv_freedevid(struct nfsdevice *ds); -static int nfsrv_setdsserver(char *dspathp, char *mirrorp, NFSPROC_T *p, +static int nfsrv_setdsserver(char *dspathp, NFSPROC_T *p, struct nfsdevice **dsp); static int nfsrv_delds(char *devid, NFSPROC_T *p); -static void nfsrv_deleteds(struct nfsdevice *fndds, struct nfsdevice *pards); +static void nfsrv_deleteds(struct nfsdevice *fndds); static void nfsrv_allocdevid(struct nfsdevice *ds, char *addr, char *dnshost); static void nfsrv_freealldevids(void); static void nfsrv_flexlayouterr(struct nfsrv_descript *nd, uint32_t *layp, @@ -6965,8 +6965,7 @@ int nfsrv_getdevinfo(char *devid, int layouttype, uint32_t *maxcnt, uint32_t *notify, int *devaddrlen, char **devaddr) { - struct nfsdevice *ds, *mds; - int done; + struct nfsdevice *ds; if ((layouttype != NFSLAYOUT_NFSV4_1_FILES && layouttype != NFSLAYOUT_FLEXFILE) || @@ -6978,21 +6977,11 @@ nfsrv_getdevinfo(char *devid, int layouttype, uint32_t * away, but the order changes in the list. As such, the lock only * needs to be held during the search through the list. */ - done = 0; NFSDDSLOCK(); TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) { - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, nfsdev_list) { - if (mds->nfsdev_nmp != NULL && NFSBCMP(devid, - mds->nfsdev_deviceid, NFSX_V4DEVICEID) == 0) { - ds = mds; - done = 1; - break; - } - } - if (done != 0) + if (NFSBCMP(devid, ds->nfsdev_deviceid, NFSX_V4DEVICEID) == 0 && + ds->nfsdev_nmp != NULL) break; - if (NFSBCMP(devid, ds->nfsdev_deviceid, NFSX_V4DEVICEID) == 0) - break; } NFSDDSUNLOCK(); if (ds == NULL) @@ -7083,11 +7072,8 @@ nfsrv_freeonedevid(struct nfsdevice *ds) static void nfsrv_freedevid(struct nfsdevice *ds) { - struct nfsdevice *mds, *nds; TAILQ_REMOVE(&nfsrv_devidhead, ds, nfsdev_list); - TAILQ_FOREACH_SAFE(mds, &ds->nfsdev_mirrors, nfsdev_list, nds) - nfsrv_freeonedevid(mds); nfsrv_freeonedevid(ds); } @@ -7195,11 +7181,10 @@ nfsrv_freealllayouts(void) * Look up the mount path for the DS server. */ static int -nfsrv_setdsserver(char *dspathp, char *mirrorp, NFSPROC_T *p, - struct nfsdevice **dsp) +nfsrv_setdsserver(char *dspathp, NFSPROC_T *p, struct nfsdevice **dsp) { struct nameidata nd; - struct nfsdevice *ds, *mds, *tds; + struct nfsdevice *ds; int error, i; char *dsdirpath; size_t dsdirsize; @@ -7230,8 +7215,6 @@ nfsrv_setdsserver(char *dspathp, char *mirrorp, NFSPRO */ *dsp = ds = malloc(sizeof(*ds) + nfsrv_dsdirsize * sizeof(vnode_t), M_NFSDSTATE, M_WAITOK | M_ZERO); - TAILQ_INIT(&ds->nfsdev_mirrors); - strcpy(ds->nfsdev_mirrorid, mirrorp); ds->nfsdev_dvp = nd.ni_vp; ds->nfsdev_nmp = VFSTONFS(nd.ni_vp->v_mount); NFSVOPUNLOCK(nd.ni_vp, 0); @@ -7264,33 +7247,8 @@ nfsrv_setdsserver(char *dspathp, char *mirrorp, NFSPRO } free(dsdirpath, M_TEMP); - /* - * Since this is done before the nfsd threads are running, locking - * isn't required. - * First, look for a mirror. If none found, link into main list. - */ - TAILQ_FOREACH(mds, &nfsrv_devidhead, nfsdev_list) { - if (strcmp(mds->nfsdev_mirrorid, mirrorp) == 0) { - TAILQ_INSERT_TAIL(&mds->nfsdev_mirrors, ds, - nfsdev_list); - atomic_add_int(&nfsrv_devidcnt, 1); - ds = NULL; - i = 1; - TAILQ_FOREACH(tds, &mds->nfsdev_mirrors, nfsdev_list) - i++; - if (i > NFSDEV_MAXMIRRORS) - error = ENXIO; - else if (i > nfsrv_maxpnfsmirror) { - nfsrv_maxpnfsmirror = i; - nfsrv_doflexfile = 1; /* Force Flex File. */ - } - break; - } - } - if (ds != NULL) { - TAILQ_INSERT_TAIL(&nfsrv_devidhead, ds, nfsdev_list); - atomic_add_int(&nfsrv_devidcnt, 1); - } + TAILQ_INSERT_TAIL(&nfsrv_devidhead, ds, nfsdev_list); + atomic_add_int(&nfsrv_devidcnt, 1); return (error); } @@ -7364,13 +7322,13 @@ nfsrv_deldsserver(char *dspathp, NFSPROC_T *p) struct nfsdevice * nfsrv_deldsnmp(struct nfsmount *nmp, NFSPROC_T *p) { - struct nfsdevice *fndds, *fndpards; + struct nfsdevice *fndds; NFSD_DEBUG(4, "deldsdvp\n"); NFSDDSLOCK(); - fndds = nfsv4_findmirror(nmp, &fndpards); + fndds = nfsv4_findmirror(nmp); if (fndds != NULL) - nfsrv_deleteds(fndds, fndpards); + nfsrv_deleteds(fndds); NFSDDSUNLOCK(); if (fndds != NULL) { nfsrv_flexmirrordel(fndds->nfsdev_deviceid, p); @@ -7388,8 +7346,9 @@ nfsrv_deldsnmp(struct nfsmount *nmp, NFSPROC_T *p) static int nfsrv_delds(char *devid, NFSPROC_T *p) { - struct nfsdevice *ds, *mds, *fndds, *fndpards; + struct nfsdevice *ds, *fndds; struct nfsmount *nmp; + int fndmirror; NFSD_DEBUG(4, "delds\n"); /* @@ -7397,48 +7356,27 @@ nfsrv_delds(char *devid, NFSPROC_T *p) * Remove the DS entry if found and there is a mirror. */ fndds = NULL; - fndpards = NULL; nmp = NULL; + fndmirror = 0; NFSDDSLOCK(); TAILQ_FOREACH(ds, &nfsrv_devidhead, nfsdev_list) { - if (fndds != NULL) - break; if (NFSBCMP(ds->nfsdev_deviceid, devid, NFSX_V4DEVICEID) == 0 && ds->nfsdev_nmp != NULL) { - /* If there are no mirrors, return error. */ - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, nfsdev_list) { - if (mds->nfsdev_nmp != NULL) - break; - } - if (mds == NULL) { - NFSDDSUNLOCK(); - NFSD_DEBUG(4, "no mirror for DS\n"); - return (ENXIO); - } NFSD_DEBUG(4, "fnd main ds\n"); fndds = ds; + } else if (ds->nfsdev_nmp != NULL) + fndmirror = 1; + if (fndds != NULL && fndmirror != 0) break; - } else { - TAILQ_FOREACH(mds, &ds->nfsdev_mirrors, nfsdev_list) { - if (NFSBCMP(mds->nfsdev_deviceid, devid, - NFSX_V4DEVICEID) == 0 && - mds->nfsdev_nmp != NULL) { - NFSD_DEBUG(4, "fnd mirror ds\n"); - fndds = mds; - fndpards = ds; - break; - } - } - } } - if (fndds != NULL) { + if (fndds != NULL && fndmirror != 0) { nmp = fndds->nfsdev_nmp; NFSLOCKMNT(nmp); if ((nmp->nm_privflag & (NFSMNTP_FORCEDISM | NFSMNTP_CANCELRPCS)) == 0) { nmp->nm_privflag |= NFSMNTP_CANCELRPCS; NFSUNLOCKMNT(nmp); - nfsrv_deleteds(fndds, fndpards); + nfsrv_deleteds(fndds); } else { NFSUNLOCKMNT(nmp); nmp = NULL; @@ -7459,56 +7397,14 @@ nfsrv_delds(char *devid, NFSPROC_T *p) } /* - * Remove a mirror from the list, moving a mirror up to be the first one, - * as required. - * The mirror is removed by setting nfsdev_nmp = NULL. + * Mark a DS as disabled by setting nfsdev_nmp = NULL. */ static void -nfsrv_deleteds(struct nfsdevice *fndds, struct nfsdevice *pards) +nfsrv_deleteds(struct nfsdevice *fndds) { - struct nfsdevice *mds, *nds; - if (pards != NULL) { - NFSD_DEBUG(4, "deleteds: deleting a mirror\n"); - fndds->nfsdev_nmp = NULL; - return; - } - - NFSD_DEBUG(4, "deleteds: deleting main ds\n"); - /* Search for a usable mirror. If none found, return. */ - TAILQ_FOREACH(mds, &fndds->nfsdev_mirrors, nfsdev_list) { - if (mds->nfsdev_nmp != NULL) - break; - } - if (mds == NULL) { - printf("nfsrv_deleteds: empty mirror\n"); - return; - } - - /* - * The fndds is the first one, so make the first valid entry in the - * mirror list the first one. - */ - TAILQ_REMOVE(&nfsrv_devidhead, fndds, nfsdev_list); - TAILQ_REMOVE(&fndds->nfsdev_mirrors, mds, nfsdev_list); - TAILQ_INIT(&mds->nfsdev_mirrors); - - /* Now, move the rest of the mirrors over to mds->nfsdev_mirrors. */ - nds = TAILQ_FIRST(&fndds->nfsdev_mirrors); - while (nds != NULL) { - NFSD_DEBUG(4, "shifting mirror up\n"); - TAILQ_REMOVE(&fndds->nfsdev_mirrors, nds, nfsdev_list); - TAILQ_INSERT_TAIL(&mds->nfsdev_mirrors, nds, nfsdev_list); - nds = TAILQ_FIRST(&fndds->nfsdev_mirrors); - } - - /* And insert mds in the first/main list. */ - TAILQ_INSERT_HEAD(&nfsrv_devidhead, mds, nfsdev_list); - - /* Put fndds in the mirror list with nfsdev_nmp == NULL. */ + NFSD_DEBUG(4, "deleteds: deleting a mirror\n"); fndds->nfsdev_nmp = NULL; - TAILQ_INIT(&fndds->nfsdev_mirrors); - TAILQ_INSERT_TAIL(&mds->nfsdev_mirrors, fndds, nfsdev_list); } /* @@ -7594,31 +7490,24 @@ int nfsrv_createdevids(struct nfsd_nfsd_args *args, NFSPROC_T *p) { struct nfsdevice *ds; - char *addrp, *dnshostp, *dspathp, *mirrorp; + char *addrp, *dnshostp, *dspathp; int error, i; addrp = args->addr; dnshostp = args->dnshost; dspathp = args->dspath; - mirrorp = args->mirror; - nfsrv_maxpnfsmirror = 1; - if (addrp == NULL || dnshostp == NULL || dspathp == NULL || - mirrorp == NULL) + nfsrv_maxpnfsmirror = args->mirrorcnt; + if (addrp == NULL || dnshostp == NULL || dspathp == NULL) return (0); /* * Loop around for each nul-terminated string in args->addr, - * args->dnshost, args->dnspath and args->mirror. + * args->dnshost and args->dnspath. */ while (addrp < (args->addr + args->addrlen) && dnshostp < (args->dnshost + args->dnshostlen) && - dspathp < (args->dspath + args->dspathlen) && - mirrorp < (args->mirror + args->mirrorlen)) { - error = 0; - if (*mirrorp == '\0' || strlen(mirrorp) > NFSDEV_MIRRORSTR) - error = ENXIO; - if (error == 0) - error = nfsrv_setdsserver(dspathp, mirrorp, p, &ds); + dspathp < (args->dspath + args->dspathlen)) { + error = nfsrv_setdsserver(dspathp, p, &ds); if (error != 0) { /* Free all DS servers. */ nfsrv_freealldevids(); @@ -7629,8 +7518,14 @@ nfsrv_createdevids(struct nfsd_nfsd_args *args, NFSPRO addrp += (strlen(addrp) + 1); dnshostp += (strlen(dnshostp) + 1); dspathp += (strlen(dspathp) + 1); - mirrorp += (strlen(mirrorp) + 1); } + if (nfsrv_devidcnt < nfsrv_maxpnfsmirror) { + /* Free all DS servers. */ + nfsrv_freealldevids(); + nfsrv_devidcnt = 0; + nfsrv_maxpnfsmirror = 1; + return (ENXIO); + } /* * Allocate the nfslayout hash table now, since this is a pNFS server. @@ -8084,7 +7979,7 @@ nfsrv_createdsfile(vnode_t vp, fhandle_t *fhp, struct } /* - * Look up the MDS file exclusively locked, and then get the extended attribute + * Look up the MDS file shared locked, and then get the extended attribute * to find the extant DS file to be copied to the new mirror. * If successful, *vpp is set to the MDS file's vp and *nvpp is * set to a DS data file for the MDS file, both exclusively locked. @@ -8093,19 +7988,24 @@ nfsrv_createdsfile(vnode_t vp, fhandle_t *fhp, struct */ int nfsrv_mdscopymr(char *mdspathp, char *dspathp, char *curdspathp, char *buf, - int *buflenp, NFSPROC_T *p, struct vnode **vpp, struct vnode **nvpp, - struct pnfsdsfile **pfp, struct nfsdevice **dsp) + int *buflenp, char *fname, NFSPROC_T *p, struct vnode **vpp, + struct vnode **nvpp, struct pnfsdsfile **pfp, struct nfsdevice **dsp, + struct nfsdevice **fdsp) { struct nameidata nd; struct vnode *vp, *curvp; struct pnfsdsfile *pf; struct nfsmount *nmp, *curnmp; - int dsdir, error, mirrorcnt, zeroippos; - char fname[PNFS_FILENAME_LEN + 1]; + int dsdir, error, mirrorcnt, ippos; vp = NULL; curvp = NULL; + curnmp = NULL; *dsp = NULL; + *fdsp = NULL; + if (dspathp == NULL && curdspathp != NULL) + return (EPERM); + /* * Look up the MDS file shared locked. The lock will be upgraded * to an exclusive lock after any rw layouts have been returned. @@ -8124,84 +8024,99 @@ nfsrv_mdscopymr(char *mdspathp, char *dspathp, char *c } vp = nd.ni_vp; - /* - * Look up the current DS path and find the nfsdev structure for it. - */ - NFSD_DEBUG(4, "curmdsdev path=%s\n", curdspathp); - NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, UIO_SYSSPACE, - curdspathp, p); - error = namei(&nd); - NFSD_DEBUG(4, "ds lookup=%d\n", error); - if (error != 0) { - vput(vp); - return (error); + if (curdspathp != NULL) { + /* + * Look up the current DS path and find the nfsdev structure for + * it. + */ + NFSD_DEBUG(4, "curmdsdev path=%s\n", curdspathp); + NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, + UIO_SYSSPACE, curdspathp, p); + error = namei(&nd); + NFSD_DEBUG(4, "ds lookup=%d\n", error); + if (error != 0) { + vput(vp); + return (error); + } + if (nd.ni_vp->v_type != VDIR) { + vput(nd.ni_vp); + vput(vp); + NFSD_DEBUG(4, "curdspath not dir\n"); + return (ENOTDIR); + } + if (strcmp(nd.ni_vp->v_mount->mnt_vfc->vfc_name, "nfs") != 0) { + vput(nd.ni_vp); + vput(vp); + NFSD_DEBUG(4, "curdspath not an NFS mount\n"); + return (ENXIO); + } + curnmp = VFSTONFS(nd.ni_vp->v_mount); + + /* Search the nfsdev list for a match. */ + NFSDDSLOCK(); + *fdsp = nfsv4_findmirror(curnmp); + NFSDDSUNLOCK(); + if (*fdsp == NULL) + curnmp = NULL; + if (curnmp == NULL) { + vput(nd.ni_vp); + vput(vp); + NFSD_DEBUG(4, "mdscopymr: no current ds\n"); + return (ENXIO); + } + curvp = nd.ni_vp; } - if (nd.ni_vp->v_type != VDIR) { - vput(nd.ni_vp); - vput(vp); - NFSD_DEBUG(4, "curdspath not dir\n"); - return (ENOTDIR); - } - if (strcmp(nd.ni_vp->v_mount->mnt_vfc->vfc_name, "nfs") != 0) { - vput(nd.ni_vp); - vput(vp); - NFSD_DEBUG(4, "curdspath not an NFS mount\n"); - return (ENXIO); - } - curnmp = VFSTONFS(nd.ni_vp->v_mount); - /* Search the nfsdev list for a match. */ - NFSDDSLOCK(); - if (nfsv4_findmirror(curnmp, NULL) == NULL) - curnmp = NULL; - NFSDDSUNLOCK(); - if (curnmp == NULL) { - vput(nd.ni_vp); - vput(vp); - NFSD_DEBUG(4, "mdscopymr: no current ds\n"); - return (ENXIO); + if (dspathp != NULL) { + /* Look up the nfsdev path and find the nfsdev structure. */ + NFSD_DEBUG(4, "mdsdev path=%s\n", dspathp); + NDINIT(&nd, LOOKUP, FOLLOW | LOCKSHARED | LOCKLEAF, + UIO_SYSSPACE, dspathp, p); + error = namei(&nd); + NFSD_DEBUG(4, "ds lookup=%d\n", error); + if (error != 0) { + vput(vp); + if (curvp != NULL) + vput(curvp); + return (error); + } + if (nd.ni_vp->v_type != VDIR || nd.ni_vp == curvp) { + vput(nd.ni_vp); + vput(vp); + if (curvp != NULL) + vput(curvp); + NFSD_DEBUG(4, "dspath not dir\n"); + if (nd.ni_vp == curvp) + return (EPERM); + return (ENOTDIR); + } + if (strcmp(nd.ni_vp->v_mount->mnt_vfc->vfc_name, "nfs") != 0) { + vput(nd.ni_vp); + vput(vp); + if (curvp != NULL) + vput(curvp); + NFSD_DEBUG(4, "dspath not an NFS mount\n"); + return (ENXIO); + } + nmp = VFSTONFS(nd.ni_vp->v_mount); + + /* Search the nfsdev list for a match. */ + NFSDDSLOCK(); + *dsp = nfsv4_findmirror(nmp); + NFSDDSUNLOCK(); + if (*dsp == NULL) { + vput(nd.ni_vp); + vput(vp); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201805290319.w4T3JuNC023690>