Date: Fri, 9 Jul 2021 00:44:00 GMT From: Rick Macklem <rmacklem@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 1e0a518d6548 - main - nfscl: Add a Linux compatible "nconnect" mount option Message-ID: <202107090044.1690i00x047695@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by rmacklem: URL: https://cgit.FreeBSD.org/src/commit/?id=1e0a518d65488caafff89a4ecba9cfb2be233379 commit 1e0a518d65488caafff89a4ecba9cfb2be233379 Author: Rick Macklem <rmacklem@FreeBSD.org> AuthorDate: 2021-07-09 00:39:04 +0000 Commit: Rick Macklem <rmacklem@FreeBSD.org> CommitDate: 2021-07-09 00:39:04 +0000 nfscl: Add a Linux compatible "nconnect" mount option Linux has had an "nconnect" NFS mount option for some time. It specifies that N (up to 16) TCP connections are to created for a mount, instead of just one TCP connection. A discussion on freebsd-net@ indicated that this could improve client<-->server network bandwidth, if either the client or server have one of the following: - multiple network ports aggregated to-gether with lagg/lacp. - a fast NIC that is using multiple queues It does result in using more IP port#s and might increase server peak load for a client. One difference from the Linux implementation is that this implementation uses the first TCP connection for all RPCs composed of small messages and uses the additional TCP connections for RPCs that normally have large messages (Read/Readdir/Write). The Linux implementation spreads all RPCs across all TCP connections in a round robin fashion, whereas this implementation spreads Read/Readdir/Write across the additional TCP connections in a round robin fashion. Reviewed by: markj MFC after: 2 weeks Differential Revision: https://reviews.freebsd.org/D30970 --- sys/fs/nfs/nfs_commonkrpc.c | 70 ++++++++++++++++++++++++++++++++++------ sys/fs/nfs/nfs_commonsubs.c | 5 +-- sys/fs/nfs/nfs_var.h | 4 +-- sys/fs/nfsclient/nfs_clrpcops.c | 8 ++--- sys/fs/nfsclient/nfs_clvfsops.c | 66 ++++++++++++++++++++++++++++++------- sys/fs/nfsclient/nfsmount.h | 8 +++++ sys/fs/nfsserver/nfs_nfsdstate.c | 6 ++-- 7 files changed, 134 insertions(+), 33 deletions(-) diff --git a/sys/fs/nfs/nfs_commonkrpc.c b/sys/fs/nfs/nfs_commonkrpc.c index 63ff02331a0e..423ddb52494f 100644 --- a/sys/fs/nfs/nfs_commonkrpc.c +++ b/sys/fs/nfs/nfs_commonkrpc.c @@ -167,7 +167,8 @@ static int nfsv2_procid[NFS_V3NPROCS] = { */ int newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp, - struct ucred *cred, NFSPROC_T *p, int callback_retry_mult, bool dotls) + struct ucred *cred, NFSPROC_T *p, int callback_retry_mult, bool dotls, + struct __rpc_client **clipp) { int rcvreserve, sndreserve; int pktscale, pktscalesav; @@ -420,15 +421,22 @@ newnfs_connect(struct nfsmount *nmp, struct nfssockreq *nrp, CLNT_CONTROL(client, CLSET_RETRY_TIMEOUT, &timo); } + /* + * *clipp is &nrp->nr_client or &nm_aconn[nmp->nm_nextaconn]. + * The latter case is for additional connections specified by the + * "nconnect" mount option. nr_mtx etc is used for these additional + * connections, as well as nr_client in the nfssockreq + * structure for the mount. + */ mtx_lock(&nrp->nr_mtx); - if (nrp->nr_client != NULL) { + if (*clipp != NULL) { mtx_unlock(&nrp->nr_mtx); /* * Someone else already connected. */ CLNT_RELEASE(client); } else { - nrp->nr_client = client; + *clipp = client; /* * Protocols that do not require connections may be optionally * left unconnected for servers that reply from a port other @@ -453,18 +461,34 @@ out: * NFS disconnect. Clean up and unlink. */ void -newnfs_disconnect(struct nfssockreq *nrp) +newnfs_disconnect(struct nfsmount *nmp, struct nfssockreq *nrp) { - CLIENT *client; + CLIENT *client, *aconn[NFS_MAXNCONN - 1]; + int i; mtx_lock(&nrp->nr_mtx); if (nrp->nr_client != NULL) { client = nrp->nr_client; nrp->nr_client = NULL; + if (nmp != NULL && nmp->nm_aconnect > 0) { + for (i = 0; i < nmp->nm_aconnect; i++) { + aconn[i] = nmp->nm_aconn[i]; + nmp->nm_aconn[i] = NULL; + } + } mtx_unlock(&nrp->nr_mtx); rpc_gss_secpurge_call(client); CLNT_CLOSE(client); CLNT_RELEASE(client); + if (nmp != NULL && nmp->nm_aconnect > 0) { + for (i = 0; i < nmp->nm_aconnect; i++) { + if (aconn[i] != NULL) { + rpc_gss_secpurge_call(aconn[i]); + CLNT_CLOSE(aconn[i]); + CLNT_RELEASE(aconn[i]); + } + } + } } else { mtx_unlock(&nrp->nr_mtx); } @@ -565,7 +589,7 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp, int error = 0, usegssname = 0, secflavour = AUTH_SYS; int freeslot, maxslot, reterr, slotpos, timeo; u_int16_t procnum; - u_int trylater_delay = 1; + u_int nextconn, trylater_delay = 1; struct nfs_feedback_arg nf; struct timeval timo; AUTH *auth; @@ -577,6 +601,7 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp, struct ucred *authcred; struct nfsclsession *sep; uint8_t sessionid[NFSX_V4SESSIONID]; + bool nextconn_set; sep = dssep; if (xidp != NULL) @@ -602,12 +627,24 @@ newnfs_request(struct nfsrv_descript *nd, struct nfsmount *nmp, } /* - * XXX if not already connected call nfs_connect now. Longer - * term, change nfs_mount to call nfs_connect unconditionally - * and let clnt_reconnect_create handle reconnects. + * If not already connected call newnfs_connect now. */ if (nrp->nr_client == NULL) - newnfs_connect(nmp, nrp, cred, td, 0, false); + newnfs_connect(nmp, nrp, cred, td, 0, false, &nrp->nr_client); + + nextconn_set = false; + if (nmp != NULL && nmp->nm_aconnect > 0 && + (nd->nd_procnum == NFSPROC_READ || + nd->nd_procnum == NFSPROC_READDIR || + nd->nd_procnum == NFSPROC_READDIRPLUS || + nd->nd_procnum == NFSPROC_WRITE)) { + nextconn = atomic_fetchadd_int(&nmp->nm_nextaconn, 1); + nextconn %= nmp->nm_aconnect; + nextconn_set = true; + if (nmp->nm_aconn[nextconn] == NULL) + newnfs_connect(nmp, nrp, cred, td, 0, false, + &nmp->nm_aconn[nextconn]); + } /* * For a client side mount, nmp is != NULL and clp == NULL. For @@ -830,6 +867,19 @@ tryagain: if (clp != NULL && sep != NULL) stat = clnt_bck_call(nrp->nr_client, &ext, procnum, nd->nd_mreq, &nd->nd_mrep, timo, sep->nfsess_xprt); + else if (nextconn_set) + /* + * When there are multiple TCP connections, send the + * RPCs with large messages on the alternate TCP + * connection(s) in a round robin fashion. + * The small RPC messages are sent on the default + * TCP connection because they do not require much + * network bandwidth and separating them from the + * large RPC messages avoids them getting "log jammed" + * behind several large RPC messages. + */ + stat = CLNT_CALL_MBUF(nmp->nm_aconn[nextconn], + &ext, procnum, nd->nd_mreq, &nd->nd_mrep, timo); else stat = CLNT_CALL_MBUF(nrp->nr_client, &ext, procnum, nd->nd_mreq, &nd->nd_mrep, timo); diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c index 02416da54f01..1bdc13123aac 100644 --- a/sys/fs/nfs/nfs_commonsubs.c +++ b/sys/fs/nfs/nfs_commonsubs.c @@ -3624,7 +3624,8 @@ nfsrv_nfsuserdport(struct nfsuserd_args *nargs, NFSPROC_T *p) } rp->nr_vers = RPCNFSUSERD_VERS; if (error == 0) - error = newnfs_connect(NULL, rp, NFSPROCCRED(p), p, 0, false); + error = newnfs_connect(NULL, rp, NFSPROCCRED(p), p, 0, false, + &rp->nr_client); if (error == 0) { NFSLOCKNAMEID(); nfsrv_nfsuserd = RUNNING; @@ -3658,7 +3659,7 @@ nfsrv_nfsuserddelport(void) msleep(&nfsrv_userdupcalls, NFSNAMEIDMUTEXPTR, PVFS, "nfsupcalls", 0); NFSUNLOCKNAMEID(); - newnfs_disconnect(&nfsrv_nfsuserdsock); + newnfs_disconnect(NULL, &nfsrv_nfsuserdsock); free(nfsrv_nfsuserdsock.nr_nam, M_SONAME); NFSLOCKNAMEID(); nfsrv_nfsuserd = NOTRUNNING; diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h index 5c474d97ad04..9cbaeae361a6 100644 --- a/sys/fs/nfs/nfs_var.h +++ b/sys/fs/nfs/nfs_var.h @@ -772,8 +772,8 @@ int newnfs_request(struct nfsrv_descript *, struct nfsmount *, struct ucred *, u_int32_t, u_int32_t, u_char *, int, u_int64_t *, struct nfsclsession *); int newnfs_connect(struct nfsmount *, struct nfssockreq *, - struct ucred *, NFSPROC_T *, int, bool); -void newnfs_disconnect(struct nfssockreq *); + struct ucred *, NFSPROC_T *, int, bool, struct __rpc_client **); +void newnfs_disconnect(struct nfsmount *, struct nfssockreq *); int newnfs_sigintr(struct nfsmount *, NFSPROC_T *); /* nfs_nfsdkrpc.c */ diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c index 128afb33b9ef..213399551097 100644 --- a/sys/fs/nfsclient/nfs_clrpcops.c +++ b/sys/fs/nfsclient/nfs_clrpcops.c @@ -5610,7 +5610,7 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in *sin, * unmount, but I did it anyhow. */ nrp->nr_cred = crhold(nmp->nm_sockreq.nr_cred); - error = newnfs_connect(nmp, nrp, NULL, p, 0, false); + error = newnfs_connect(nmp, nrp, NULL, p, 0, false, &nrp->nr_client); NFSCL_DEBUG(3, "DS connect=%d\n", error); dsp = NULL; @@ -5628,7 +5628,7 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in *sin, } while (error == NFSERR_MINORVERMISMATCH && firsttry++ == 0); if (error != 0) - newnfs_disconnect(nrp); + newnfs_disconnect(NULL, nrp); } else { dsp = malloc(sizeof(struct nfsclds), M_NFSCLDS, M_WAITOK | M_ZERO); @@ -5656,7 +5656,7 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in *sin, * If there is already a session for this * server, use it. */ - (void)newnfs_disconnect(nrp); + newnfs_disconnect(NULL, nrp); nfscl_freenfsclds(dsp); *dspp = tdsp; return (0); @@ -5688,7 +5688,7 @@ nfsrpc_fillsa(struct nfsmount *nmp, struct sockaddr_in *sin, NFSUNLOCKMNT(nmp); *dspp = dsp; } else if (dsp != NULL) { - newnfs_disconnect(nrp); + newnfs_disconnect(NULL, nrp); nfscl_freenfsclds(dsp); } return (error); diff --git a/sys/fs/nfsclient/nfs_clvfsops.c b/sys/fs/nfsclient/nfs_clvfsops.c index 931f37d4ef67..57eae3f1f1cf 100644 --- a/sys/fs/nfsclient/nfs_clvfsops.c +++ b/sys/fs/nfsclient/nfs_clvfsops.c @@ -118,7 +118,7 @@ static void nfs_decode_args(struct mount *mp, struct nfsmount *nmp, static int mountnfs(struct nfs_args *, struct mount *, struct sockaddr *, char *, u_char *, int, u_char *, int, u_char *, int, struct vnode **, struct ucred *, - struct thread *, int, int, int, uint32_t, char *); + struct thread *, int, int, int, uint32_t, char *, int); static void nfs_getnlminfo(struct vnode *, uint8_t *, size_t *, struct sockaddr_storage *, int *, off_t *, struct timeval *); @@ -548,7 +548,7 @@ nfs_mountdiskless(char *path, nam = sodupsockaddr((struct sockaddr *)sin, M_WAITOK); if ((error = mountnfs(args, mp, nam, path, NULL, 0, dirpath, dirlen, NULL, 0, vpp, td->td_ucred, td, NFS_DEFAULT_NAMETIMEO, - NFS_DEFAULT_NEGNAMETIMEO, 0, 0, NULL)) != 0) { + NFS_DEFAULT_NEGNAMETIMEO, 0, 0, NULL, 0)) != 0) { printf("nfs_mountroot: mount %s on /: %d\n", path, error); return (error); } @@ -715,14 +715,14 @@ nfs_decode_args(struct mount *mp, struct nfsmount *nmp, struct nfs_args *argp, haslock = 1; } if (!error) { - newnfs_disconnect(&nmp->nm_sockreq); + newnfs_disconnect(nmp, &nmp->nm_sockreq); if (haslock) newnfs_sndunlock(&nmp->nm_sockreq.nr_lock); nmp->nm_sotype = argp->sotype; nmp->nm_soproto = argp->proto; if (nmp->nm_sotype == SOCK_DGRAM) while (newnfs_connect(nmp, &nmp->nm_sockreq, - cred, td, 0, false)) { + cred, td, 0, false, &nmp->nm_sockreq.nr_client)) { printf("newnfs_args: retrying connect\n"); (void) nfs_catnap(PSOCK, 0, "nfscon"); } @@ -750,7 +750,7 @@ static const char *nfs_opts[] = { "from", "nfs_args", "resvport", "readahead", "hostname", "timeo", "timeout", "addr", "fh", "nfsv3", "sec", "principal", "nfsv4", "gssname", "allgssname", "dirpath", "minorversion", "nametimeo", "negnametimeo", "nocto", "noncontigwr", - "pnfs", "wcommitsize", "oneopenown", "tls", "tlscertname", + "pnfs", "wcommitsize", "oneopenown", "tls", "tlscertname", "nconnect", NULL }; /* @@ -902,6 +902,7 @@ nfs_mount(struct mount *mp) krbnamelen, srvkrbnamelen; size_t hstlen; uint32_t newflag; + int aconn = 0; has_nfs_args_opt = 0; has_nfs_from_opt = 0; @@ -1192,6 +1193,20 @@ nfs_mount(struct mount *mp) goto out; } } + if (vfs_getopt(mp->mnt_optnew, "nconnect", (void **)&opt, NULL) == + 0) { + ret = sscanf(opt, "%d", &aconn); + if (ret != 1 || aconn < 1 || aconn > NFS_MAXNCONN) { + vfs_mount_error(mp, "illegal nconnect: %s", opt); + error = EINVAL; + goto out; + } + /* + * Setting nconnect=1 is a no-op, allowed so that + * the option can be used in a Linux compatible way. + */ + aconn--; + } if (vfs_getopt(mp->mnt_optnew, "sec", (void **) &secname, NULL) == 0) nfs_sec_name(secname, &args.flags); @@ -1359,10 +1374,25 @@ nfs_mount(struct mount *mp) } } + if (aconn > 0 && (args.sotype != SOCK_STREAM || + (args.flags & NFSMNT_NFSV4) == 0 || minvers == 0)) { + /* + * RFC 5661 requires that an NFSv4.1/4.2 server + * send an RPC reply on the same TCP connection + * as the one it received the request on. + * This property in required for "nconnect" and + * might not be the case for NFSv3 or NFSv4.0 servers. + */ + vfs_mount_error(mp, "nconnect should only be used " + "for NFSv4.1/4.2 mounts"); + error = EINVAL; + goto out; + } + args.fh = nfh; error = mountnfs(&args, mp, nam, hst, krbname, krbnamelen, dirpath, dirlen, srvkrbname, srvkrbnamelen, &vp, td->td_ucred, td, - nametimeo, negnametimeo, minvers, newflag, tlscertname); + nametimeo, negnametimeo, minvers, newflag, tlscertname, aconn); out: if (!error) { MNT_ILOCK(mp); @@ -1410,7 +1440,7 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam, char *hst, u_char *krbname, int krbnamelen, u_char *dirpath, int dirlen, u_char *srvkrbname, int srvkrbnamelen, struct vnode **vpp, struct ucred *cred, struct thread *td, int nametimeo, int negnametimeo, - int minvers, uint32_t newflag, char *tlscertname) + int minvers, uint32_t newflag, char *tlscertname, int aconn) { struct nfsmount *nmp; struct nfsnode *np; @@ -1577,7 +1607,8 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam, else nmp->nm_sockreq.nr_vers = NFS_VER2; - if ((error = newnfs_connect(nmp, &nmp->nm_sockreq, cred, td, 0, false))) + if ((error = newnfs_connect(nmp, &nmp->nm_sockreq, cred, td, 0, false, + &nmp->nm_sockreq.nr_client))) goto bad; /* For NFSv4, get the clientid now. */ if ((argp->flags & NFSMNT_NFSV4) != 0) { @@ -1586,6 +1617,12 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam, NFSCL_DEBUG(3, "aft getcl=%d\n", error); if (error != 0) goto bad; + if (aconn > 0 && nmp->nm_minorvers == 0) { + vfs_mount_error(mp, "nconnect should only be used " + "for NFSv4.1/4.2 mounts"); + error = EINVAL; + goto bad; + } } if (nmp->nm_fhsize == 0 && (nmp->nm_flag & NFSMNT_NFSV4) && @@ -1680,6 +1717,10 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam, MNT_IUNLOCK(mp); } + /* Can now allow additional connections. */ + if (aconn > 0) + nmp->nm_aconnect = aconn; + /* * Lose the lock but keep the ref. */ @@ -1692,7 +1733,7 @@ mountnfs(struct nfs_args *argp, struct mount *mp, struct sockaddr *nam, bad: if (clp != NULL) nfscl_clientrelease(clp); - newnfs_disconnect(&nmp->nm_sockreq); + newnfs_disconnect(NULL, &nmp->nm_sockreq); crfree(nmp->nm_sockreq.nr_cred); if (nmp->nm_sockreq.nr_auth != NULL) AUTH_DESTROY(nmp->nm_sockreq.nr_auth); @@ -1707,7 +1748,7 @@ bad: TAILQ_FOREACH_SAFE(dsp, &nmp->nm_sess, nfsclds_list, tdsp) { if (dsp != TAILQ_FIRST(&nmp->nm_sess) && dsp->nfsclds_sockp != NULL) - newnfs_disconnect(dsp->nfsclds_sockp); + newnfs_disconnect(NULL, dsp->nfsclds_sockp); nfscl_freenfsclds(dsp); } free(nmp->nm_tlscertname, M_NEWNFSMNT); @@ -1793,7 +1834,7 @@ nfs_unmount(struct mount *mp, int mntflags) msleep(nmp, &nmp->nm_mtx, PVFS, "nfsfdism", 0); mtx_unlock(&nmp->nm_mtx); - newnfs_disconnect(&nmp->nm_sockreq); + newnfs_disconnect(nmp, &nmp->nm_sockreq); crfree(nmp->nm_sockreq.nr_cred); free(nmp->nm_nam, M_SONAME); if (nmp->nm_sockreq.nr_auth != NULL) @@ -1803,7 +1844,7 @@ nfs_unmount(struct mount *mp, int mntflags) TAILQ_FOREACH_SAFE(dsp, &nmp->nm_sess, nfsclds_list, tdsp) { if (dsp != TAILQ_FIRST(&nmp->nm_sess) && dsp->nfsclds_sockp != NULL) - newnfs_disconnect(dsp->nfsclds_sockp); + newnfs_disconnect(NULL, dsp->nfsclds_sockp); nfscl_freenfsclds(dsp); } free(nmp->nm_tlscertname, M_NEWNFSMNT); @@ -2067,6 +2108,7 @@ void nfscl_retopts(struct nfsmount *nmp, char *buffer, size_t buflen) &blen); nfscl_printopt(nmp, (nmp->nm_flag & NFSMNT_NOCONN) != 0, ",noconn", &buf, &blen); + nfscl_printoptval(nmp, nmp->nm_aconnect + 1, ",nconnect", &buf, &blen); nfscl_printopt(nmp, (nmp->nm_flag & NFSMNT_SOFT) == 0, ",hard", &buf, &blen); nfscl_printopt(nmp, (nmp->nm_flag & NFSMNT_SOFT) != 0, ",soft", &buf, diff --git a/sys/fs/nfsclient/nfsmount.h b/sys/fs/nfsclient/nfsmount.h index f8ea8c9a1418..a5997e474be9 100644 --- a/sys/fs/nfsclient/nfsmount.h +++ b/sys/fs/nfsclient/nfsmount.h @@ -39,6 +39,9 @@ #include <nfs/nfs_mountcommon.h> +/* Maximum value for nm_nconnect. */ +#define NFS_MAXNCONN 16 + /* * Mount structure. * One allocated on every NFS mount. @@ -81,6 +84,11 @@ struct nfsmount { u_int64_t nm_clval; /* identifies which clientid */ u_int64_t nm_fsid[2]; /* NFSv4 fsid */ int nm_minorvers; /* Minor version # for NFSv4 */ + u_int nm_aconnect; /* additional TCP connections */ + u_int nm_nextaconn; /* Next nm_aconn[] to use */ + /* unclipped, wraps to 0 */ + struct __rpc_client *nm_aconn[NFS_MAXNCONN - 1]; /* Additional nconn */ + /* Locked via nm_sockreq.nr_mtx */ u_int16_t nm_krbnamelen; /* Krb5 host principal, if any */ u_int16_t nm_dirpathlen; /* and mount dirpath, for V4 */ u_int16_t nm_srvkrbnamelen; /* and the server's target name */ diff --git a/sys/fs/nfsserver/nfs_nfsdstate.c b/sys/fs/nfsserver/nfs_nfsdstate.c index 98f5e26d49bb..01280c8e49c6 100644 --- a/sys/fs/nfsserver/nfs_nfsdstate.c +++ b/sys/fs/nfsserver/nfs_nfsdstate.c @@ -1368,7 +1368,7 @@ nfsrv_zapclient(struct nfsclient *clp, NFSPROC_T *p) NULL, 0, NULL, NULL, NULL, 0, p); } #endif - newnfs_disconnect(&clp->lc_req); + newnfs_disconnect(NULL, &clp->lc_req); free(clp->lc_req.nr_nam, M_SONAME); NFSFREEMUTEX(&clp->lc_req.nr_mtx); free(clp->lc_stateid, M_NFSDCLIENT); @@ -4577,10 +4577,10 @@ nfsrv_docallback(struct nfsclient *clp, int procnum, nfsv4stateid_t *stateidp, nfsrv_freesession(sep, NULL); } else if (nd->nd_procnum == NFSV4PROC_CBNULL) error = newnfs_connect(NULL, &clp->lc_req, cred, - NULL, 1, dotls); + NULL, 1, dotls, &clp->lc_req.nr_client); else error = newnfs_connect(NULL, &clp->lc_req, cred, - NULL, 3, dotls); + NULL, 3, dotls, &clp->lc_req.nr_client); } newnfs_sndunlock(&clp->lc_req.nr_lock); NFSD_DEBUG(4, "aft sndunlock=%d\n", error);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202107090044.1690i00x047695>