From owner-svn-src-projects@freebsd.org Mon Oct 16 23:43:19 2017
Return-Path: <owner-svn-src-projects@freebsd.org>
Delivered-To: svn-src-projects@mailman.ysv.freebsd.org
Received: from mx1.freebsd.org (mx1.freebsd.org
[IPv6:2001:1900:2254:206a::19:1])
by mailman.ysv.freebsd.org (Postfix) with ESMTP id D88A6E48997
for <svn-src-projects@mailman.ysv.freebsd.org>;
Mon, 16 Oct 2017 23:43:19 +0000 (UTC)
(envelope-from rmacklem@FreeBSD.org)
Received: from repo.freebsd.org (repo.freebsd.org
[IPv6:2610:1c1:1:6068::e6a:0])
(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))
(Client did not present a certificate)
by mx1.freebsd.org (Postfix) with ESMTPS id B198872815;
Mon, 16 Oct 2017 23:43:19 +0000 (UTC)
(envelope-from rmacklem@FreeBSD.org)
Received: from repo.freebsd.org ([127.0.1.37])
by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id v9GNhIDB009491;
Mon, 16 Oct 2017 23:43:18 GMT (envelope-from rmacklem@FreeBSD.org)
Received: (from rmacklem@localhost)
by repo.freebsd.org (8.15.2/8.15.2/Submit) id v9GNhIdh009490;
Mon, 16 Oct 2017 23:43:18 GMT (envelope-from rmacklem@FreeBSD.org)
Message-Id: <201710162343.v9GNhIdh009490@repo.freebsd.org>
X-Authentication-Warning: repo.freebsd.org: rmacklem set sender to
rmacklem@FreeBSD.org using -f
From: Rick Macklem <rmacklem@FreeBSD.org>
Date: Mon, 16 Oct 2017 23:43:18 +0000 (UTC)
To: src-committers@freebsd.org, svn-src-projects@freebsd.org
Subject: svn commit: r324678 - projects/pnfs-planb-server/sys/fs/nfsserver
X-SVN-Group: projects
X-SVN-Commit-Author: rmacklem
X-SVN-Commit-Paths: projects/pnfs-planb-server/sys/fs/nfsserver
X-SVN-Commit-Revision: 324678
X-SVN-Commit-Repository: base
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
X-BeenThere: svn-src-projects@freebsd.org
X-Mailman-Version: 2.1.23
Precedence: list
List-Id: "SVN commit messages for the src " projects"
tree" <svn-src-projects.freebsd.org>
List-Unsubscribe: <https://lists.freebsd.org/mailman/options/svn-src-projects>,
<mailto:svn-src-projects-request@freebsd.org?subject=unsubscribe>
List-Archive: <http://lists.freebsd.org/pipermail/svn-src-projects/>
List-Post: <mailto:svn-src-projects@freebsd.org>
List-Help: <mailto:svn-src-projects-request@freebsd.org?subject=help>
List-Subscribe: <https://lists.freebsd.org/mailman/listinfo/svn-src-projects>,
<mailto:svn-src-projects-request@freebsd.org?subject=subscribe>
X-List-Received-Date: Mon, 16 Oct 2017 23:43:20 -0000
Author: rmacklem
Date: Mon Oct 16 23:43:18 2017
New Revision: 324678
URL: https://svnweb.freebsd.org/changeset/base/324678
Log:
Change the pNFS server to use the taskqueue code (common with the client)
to do concurrent I/O operations to the mirrored DSs, instead of using
kproc_create()/kproc_exit() for every RPC.
Modified:
projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c
Modified: projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c
==============================================================================
--- projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Mon Oct 16 23:40:24 2017 (r324677)
+++ projects/pnfs-planb-server/sys/fs/nfsserver/nfs_nfsdport.c Mon Oct 16 23:43:18 2017 (r324678)
@@ -72,6 +72,7 @@ extern struct mtx nfsrv_dsrmlock_mtx;
extern struct mtx nfsrv_dwrpclock_mtx;
extern struct mtx nfsrv_dsrpclock_mtx;
extern struct mtx nfsrv_darpclock_mtx;
+extern int nfs_pnfsiothreads;
struct vfsoptlist nfsv4root_opt, nfsv4root_newopt;
NFSDLOCKMUTEX;
NFSSTATESPINLOCK;
@@ -127,6 +128,8 @@ static int nfsrv_putfhname(fhandle_t *, char *);
static int nfsrv_pnfslookupds(struct vnode *, struct pnfsdsfile *,
struct vnode *, NFSPROC_T *);
+int nfs_pnfsio(task_fn_t *, void *);
+
SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW, 0, "NFS server");
SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW,
&nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points");
@@ -4436,6 +4439,9 @@ nfsmout:
* so that this function can be executed by a separate kernel process.
*/
struct nfsrvwritedsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
fhandle_t fh;
off_t off;
int len;
@@ -4443,7 +4449,6 @@ struct nfsrvwritedsdorpc {
struct ucred *cred;
NFSPROC_T *p;
struct mbuf *m;
- int haskproc;
int err;
};
@@ -4560,18 +4565,15 @@ nfsmout:
* Start up the thread that will execute nfsrv_writedsdorpc().
*/
static void
-start_writedsdorpc(void *arg)
+start_writedsdorpc(void *arg, int pending)
{
struct nfsrvwritedsdorpc *drpc;
drpc = (struct nfsrvwritedsdorpc *)arg;
drpc->err = nfsrv_writedsdorpc(drpc->nmp, &drpc->fh, drpc->off,
drpc->len, NULL, drpc->m, drpc->cred, drpc->p);
- NFSDWRPCLOCK();
- drpc->haskproc = 0;
- wakeup(drpc);
- NFSDWRPCUNLOCK();
- kproc_exit(0);
+ drpc->done = 1;
+ NFSD_DEBUG(4, "start_writedsdorpc: err=%d\n", drpc->err);
}
static int
@@ -4582,7 +4584,7 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s
struct nfsrvwritedsdorpc *drpc, *tdrpc;
struct nfsvattr na;
struct mbuf *m;
- int error, haskproc, i, offs, ret;
+ int error, i, offs, ret, timo;
NFSD_DEBUG(4, "in nfsrv_writedsrpc\n");
KASSERT(*mpp != NULL, ("nfsrv_writedsrpc: NULL mbuf chain"));
@@ -4599,23 +4601,25 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s
* Do the write RPC for every DS, using a separate kernel process
* for every DS except the last one.
*/
- haskproc = 0;
error = 0;
for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ tdrpc->done = 0;
tdrpc->fh = *fhp;
tdrpc->off = off;
tdrpc->len = len;
tdrpc->nmp = *nmpp;
tdrpc->cred = cred;
tdrpc->p = p;
+ tdrpc->inprog = 0;
+ tdrpc->err = 0;
tdrpc->m = m_copym(*mpp, offs, NFSM_RNDUP(len), M_WAITOK);
- tdrpc->haskproc = 1;
- ret = kproc_create(start_writedsdorpc, (void *)tdrpc, NULL, 0,
- 0, "nfsdpw");
- if (ret == 0)
- haskproc = 1;
- else {
- tdrpc->haskproc = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_writedsdorpc, tdrpc);
+ NFSD_DEBUG(4, "nfsrv_writedsrpc: nfs_pnfsio=%d\n",
+ ret);
+ }
+ if (ret != 0) {
ret = nfsrv_writedsdorpc(*nmpp, fhp, off, len, NULL,
tdrpc->m, cred, p);
if (error == 0 && ret != 0)
@@ -4632,17 +4636,16 @@ nfsrv_writedsrpc(fhandle_t *fhp, off_t off, int len, s
error = nfsrv_setextattr(vp, &na, p);
NFSD_DEBUG(4, "nfsrv_writedsrpc: aft setextat=%d\n",
error);
- if (haskproc != 0) {
- /* Wait for kernel proc(s) to complete. */
- NFSDWRPCLOCK();
- for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
- while (tdrpc->haskproc != 0)
- mtx_sleep(tdrpc, NFSDWRPCLOCKMUTEXPTR, PVFS,
- "nfspw", 0);
- if (error == 0 && tdrpc->err != 0)
- error = tdrpc->err;
- }
- NFSDWRPCUNLOCK();
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ /* Wait for RPCs on separate threads to complete. */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "srvwrds", timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
}
free(drpc, M_TEMP);
return (error);
@@ -4732,6 +4735,9 @@ nfsmout:
}
struct nfsrvsetattrdsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
fhandle_t fh;
struct nfsmount *nmp;
struct vnode *vp;
@@ -4739,7 +4745,6 @@ struct nfsrvsetattrdsdorpc {
NFSPROC_T *p;
struct nfsvattr na;
struct nfsvattr dsna;
- int haskproc;
int err;
};
@@ -4747,18 +4752,14 @@ struct nfsrvsetattrdsdorpc {
* Start up the thread that will execute nfsrv_setattrdsdorpc().
*/
static void
-start_setattrdsdorpc(void *arg)
+start_setattrdsdorpc(void *arg, int pending)
{
struct nfsrvsetattrdsdorpc *drpc;
drpc = (struct nfsrvsetattrdsdorpc *)arg;
drpc->err = nfsrv_setattrdsdorpc(&drpc->fh, drpc->cred, drpc->p,
drpc->vp, drpc->nmp, &drpc->na, &drpc->dsna);
- NFSDSRPCLOCK();
- drpc->haskproc = 0;
- wakeup(drpc);
- NFSDSRPCUNLOCK();
- kproc_exit(0);
+ drpc->done = 1;
}
static int
@@ -4768,7 +4769,7 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred,
{
struct nfsrvsetattrdsdorpc *drpc, *tdrpc;
struct nfsvattr na;
- int error, haskproc, i, ret;
+ int error, i, ret, timo;
NFSD_DEBUG(4, "in nfsrv_setattrdsrpc\n");
drpc = NULL;
@@ -4780,22 +4781,24 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred,
* Do the setattr RPC for every DS, using a separate kernel process
* for every DS except the last one.
*/
- haskproc = 0;
error = 0;
for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ tdrpc->done = 0;
+ tdrpc->inprog = 0;
tdrpc->fh = *fhp;
tdrpc->nmp = *nmpp;
tdrpc->vp = vp;
tdrpc->cred = cred;
tdrpc->p = p;
tdrpc->na = *nap;
- tdrpc->haskproc = 1;
- ret = kproc_create(start_setattrdsdorpc, (void *)tdrpc, NULL, 0,
- 0, "nfsdps");
- if (ret == 0)
- haskproc = 1;
- else {
- tdrpc->haskproc = 0;
+ tdrpc->err = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_setattrdsdorpc, tdrpc);
+ NFSD_DEBUG(4, "nfsrv_setattrdsrpc: nfs_pnfsio=%d\n",
+ ret);
+ }
+ if (ret != 0) {
ret = nfsrv_setattrdsdorpc(fhp, cred, p, vp, *nmpp, nap,
&na);
if (error == 0 && ret != 0)
@@ -4810,17 +4813,16 @@ nfsrv_setattrdsrpc(fhandle_t *fhp, struct ucred *cred,
if (error == 0)
error = nfsrv_setextattr(vp, &na, p);
NFSD_DEBUG(4, "nfsrv_setattrdsrpc: aft setextat=%d\n", error);
- if (haskproc != 0) {
- /* Wait for kernel proc(s) to complete. */
- NFSDSRPCLOCK();
- for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
- while (tdrpc->haskproc != 0)
- mtx_sleep(tdrpc, NFSDSRPCLOCKMUTEXPTR, PVFS,
- "nfsps", 0);
- if (error == 0 && tdrpc->err != 0)
- error = tdrpc->err;
- }
- NFSDSRPCUNLOCK();
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ /* Wait for RPCs on separate threads to complete. */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "srvsads", timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
}
free(drpc, M_TEMP);
return (error);
@@ -4877,13 +4879,15 @@ nfsrv_setacldsdorpc(fhandle_t *fhp, struct ucred *cred
}
struct nfsrvsetacldsdorpc {
+ int done;
+ int inprog;
+ struct task tsk;
fhandle_t fh;
struct nfsmount *nmp;
struct vnode *vp;
struct ucred *cred;
NFSPROC_T *p;
struct acl *aclp;
- int haskproc;
int err;
};
@@ -4891,18 +4895,14 @@ struct nfsrvsetacldsdorpc {
* Start up the thread that will execute nfsrv_setacldsdorpc().
*/
static void
-start_setacldsdorpc(void *arg)
+start_setacldsdorpc(void *arg, int pending)
{
struct nfsrvsetacldsdorpc *drpc;
drpc = (struct nfsrvsetacldsdorpc *)arg;
drpc->err = nfsrv_setacldsdorpc(&drpc->fh, drpc->cred, drpc->p,
drpc->vp, drpc->nmp, drpc->aclp);
- NFSDARPCLOCK();
- drpc->haskproc = 0;
- wakeup(drpc);
- NFSDARPCUNLOCK();
- kproc_exit(0);
+ drpc->done = 1;
}
static int
@@ -4910,7 +4910,7 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred,
struct vnode *vp, struct nfsmount **nmpp, int mirrorcnt, struct acl *aclp)
{
struct nfsrvsetacldsdorpc *drpc, *tdrpc;
- int error, haskproc, i, ret;
+ int error, i, ret, timo;
NFSD_DEBUG(4, "in nfsrv_setacldsrpc\n");
drpc = NULL;
@@ -4922,22 +4922,24 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred,
* Do the setattr RPC for every DS, using a separate kernel process
* for every DS except the last one.
*/
- haskproc = 0;
error = 0;
for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ tdrpc->done = 0;
+ tdrpc->inprog = 0;
tdrpc->fh = *fhp;
tdrpc->nmp = *nmpp;
tdrpc->vp = vp;
tdrpc->cred = cred;
tdrpc->p = p;
tdrpc->aclp = aclp;
- tdrpc->haskproc = 1;
- ret = kproc_create(start_setacldsdorpc, (void *)tdrpc, NULL, 0,
- 0, "nfsdpa");
- if (ret == 0)
- haskproc = 1;
- else {
- tdrpc->haskproc = 0;
+ tdrpc->err = 0;
+ ret = EIO;
+ if (nfs_pnfsiothreads > 0) {
+ ret = nfs_pnfsio(start_setacldsdorpc, tdrpc);
+ NFSD_DEBUG(4, "nfsrv_setacldsrpc: nfs_pnfsio=%d\n",
+ ret);
+ }
+ if (ret != 0) {
ret = nfsrv_setacldsdorpc(fhp, cred, p, vp, *nmpp,
aclp);
if (error == 0 && ret != 0)
@@ -4950,17 +4952,16 @@ nfsrv_setacldsrpc(fhandle_t *fhp, struct ucred *cred,
if (error == 0 && ret != 0)
error = ret;
NFSD_DEBUG(4, "nfsrv_setacldsrpc: aft setextat=%d\n", error);
- if (haskproc != 0) {
- /* Wait for kernel proc(s) to complete. */
- NFSDARPCLOCK();
- for (tdrpc = drpc, i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
- while (tdrpc->haskproc != 0)
- mtx_sleep(tdrpc, NFSDARPCLOCKMUTEXPTR, PVFS,
- "nfspa", 0);
- if (error == 0 && tdrpc->err != 0)
- error = tdrpc->err;
- }
- NFSDARPCUNLOCK();
+ tdrpc = drpc;
+ timo = hz / 50; /* Wait for 20msec. */
+ if (timo < 1)
+ timo = 1;
+ for (i = 0; i < mirrorcnt - 1; i++, tdrpc++) {
+ /* Wait for RPCs on separate threads to complete. */
+ while (tdrpc->inprog != 0 && tdrpc->done == 0)
+ tsleep(&tdrpc->tsk, PVFS, "srvacds", timo);
+ if (error == 0 && tdrpc->err != 0)
+ error = tdrpc->err;
}
free(drpc, M_TEMP);
return (error);