Date: Tue, 17 Aug 2010 19:42:41 +0400 From: pluknet <pluknet@gmail.com> To: Kostik Belousov <kostikbel@gmail.com> Cc: FreeBSD Current <freebsd-current@freebsd.org> Subject: Re: LOR on nfs: vfs_vnops.c:301 kern_descrip.c:1580 Message-ID: <AANLkTimfZ=UCiX9eo9XCwcROwViFCxJoB-rD1JxdAt2B@mail.gmail.com> In-Reply-To: <20100816185456.GU2396@deviant.kiev.zoral.com.ua> References: <AANLkTimJ=d06D2z24QyRQ98zEa1Pemk4=vkNGLNiX90N@mail.gmail.com> <AANLkTi=22WWy8Z5VWqri1p70iRzyocD9bWvNHwJavZ%2Bj@mail.gmail.com> <20100816185456.GU2396@deviant.kiev.zoral.com.ua>
index | next in thread | previous in thread | raw e-mail
[-- Attachment #1 --]
2010/8/16 Kostik Belousov <kostikbel@gmail.com>:
> On Mon, Aug 16, 2010 at 09:07:24PM +0400, pluknet wrote:
>> On 16 August 2010 21:05, pluknet <pluknet@gmail.com> wrote:
>> > Hi.
>> >
>> > Seeing on mostly idle, recently updated current, while closing a file.
>> > Presumably never reported on ML.
[...]
>>
> Both LORs are valid. The fork performed deep inside the VFS call stack
> is obviously problematic. As a workaround, you may fix the number of
> nfsiods.
>
> Proper fix might consist of creating a shepherd thread which only task
> is to act on the requests on creating new nfsiods.
>
> Would you try to implement this ? I will provide the assistance, if needed.
Hmm.. I tried to move kproc_create() under shepherd thread and now stuck
with cp process lockup in [bo_wwait] when cp'ing something on nfs: cp a b.
Did I screw up something?
See weird draft patch attached (weird, as I have no idea how to nicely
exchange data between nfs_nfsiodnew() and shep_thread() thread).
load: 1.34 cmd: cp 1348 [bo_wwait] 4.74r 0.00u 0.00s 0% 1204k
tst-web# procstat -k 1348
PID TID COMM TDNAME KSTACK
1348 100095 cp - mi_switch sleepq_switch
sleepq_wait _sleep bufobj_wwait nfs_flush nfs_close vn_close
vn_closefile _fdrop closef kern_close syscallenter syscall
Xfast_syscall
Process 1347 (cp) thread 0xffffff0002ed7000 (100094)
exclusive lockmgr nfs (nfs) r = 0 (0xffffff006a05a638) locked @
/usr/src/sys/kern/vfs_vnops.c:301
(kgdb) bt
#0 sched_switch (td=0xffffff0002ed7000, newtd=0xffffffff80ca17e0,
flags=Variable "flags" is not available.
)
at /usr/src/sys/kern/sched_ule.c:1848
#1 0xffffffff805bf49b in mi_switch (flags=260, newtd=0x0)
at /usr/src/sys/kern/kern_synch.c:449
#2 0xffffffff805f50e3 in sleepq_switch (wchan=0xffffff006a05a720, pri=77)
at /usr/src/sys/kern/subr_sleepqueue.c:530
#3 0xffffffff805f5ccd in sleepq_wait (wchan=0xffffff006a05a720, pri=77)
at /usr/src/sys/kern/subr_sleepqueue.c:609
#4 0xffffffff805bfac9 in _sleep (ident=0xffffff006a05a720,
lock=0xffffff006a05a6c0, priority=Variable "priority" is not available.
) at /usr/src/sys/kern/kern_synch.c:234
#5 0xffffffff80633083 in bufobj_wwait (bo=0xffffff006a05a6c0,
slpflag=Variable "slpflag" is not available.
)
at /usr/src/sys/kern/vfs_bio.c:4016
#6 0xffffffff8077f5af in nfs_flush (vp=0xffffff006a05a5a0, waitfor=1,
commit=Variable "commit" is not available.
)
at /usr/src/sys/nfsclient/nfs_vnops.c:3216
#7 0xffffffff807802e3 in nfs_close (ap=0xffffff8029bd8900)
at /usr/src/sys/nfsclient/nfs_vnops.c:644
#8 0xffffffff8065b3fe in vn_close (vp=0xffffff006a05a5a0, flags=2,
file_cred=0xffffff006a01b600, td=0xffffff0002ed7000) at vnode_if.h:225
#9 0xffffffff8065b4fa in vn_closefile (fp=0xffffff00027c7500,
td=0xffffff0002ed7000) at /usr/src/sys/kern/vfs_vnops.c:942
#10 0xffffffff8057e3e3 in _fdrop (fp=0xffffff00027c7500, td=Variable
"td" is not available.
) at file.h:277
#11 0xffffffff8057ff4b in closef (fp=0xffffff00027c7500, td=0xffffff0002ed7000)
at /usr/src/sys/kern/kern_descrip.c:2117
---Type <return> to continue, or q <return> to quit---
#12 0xffffffff80580530 in kern_close (td=0xffffff0002ed7000, fd=4)
at /usr/src/sys/kern/kern_descrip.c:1162
--
wbr,
pluknet
[-- Attachment #2 --]
Index: sys/nfsclient/nfs_vnops.c
===================================================================
--- sys/nfsclient/nfs_vnops.c (revision 211279)
+++ sys/nfsclient/nfs_vnops.c (working copy)
@@ -212,6 +212,7 @@
* Global variables
*/
struct mtx nfs_iod_mtx;
+struct mtx shep_mtx;
enum nfsiod_state nfs_iodwant[NFS_MAXASYNCDAEMON];
struct nfsmount *nfs_iodmount[NFS_MAXASYNCDAEMON];
int nfs_numasync = 0;
Index: sys/nfsclient/nfs_subs.c
===================================================================
--- sys/nfsclient/nfs_subs.c (revision 211279)
+++ sys/nfsclient/nfs_subs.c (working copy)
@@ -354,6 +354,7 @@
*/
mtx_init(&nfs_iod_mtx, "NFS iod lock", NULL, MTX_DEF);
mtx_init(&nfs_xid_mtx, "NFS xid lock", NULL, MTX_DEF);
+ mtx_init(&shep_mtx, "NFS shep_thread lock", NULL, MTX_DEF);
nfs_pbuf_freecnt = nswbuf / 2 + 1;
Index: sys/nfsclient/nfs_nfsiod.c
===================================================================
--- sys/nfsclient/nfs_nfsiod.c (revision 211279)
+++ sys/nfsclient/nfs_nfsiod.c (working copy)
@@ -75,6 +75,13 @@
static void nfssvc_iod(void *);
+struct shepherd_str {
+ int *shep_inst;
+ int shep_niod;
+ int shep_error;
+ int shep_done;
+} shep_chan;
+
static int nfs_asyncdaemon[NFS_MAXASYNCDAEMON];
SYSCTL_DECL(_vfs_nfs);
@@ -159,10 +166,32 @@
sizeof (nfs_iodmax), sysctl_iodmax, "IU",
"Max number of nfsiod kthreads");
+static void
+shep_thread(void *dummy)
+{
+ int err = 0;
+
+// mtx_lock(&shep_mtx);
+ for (;;) {
+// err = msleep(&shep_chan, &shep_mtx, PWAIT | PCATCH, "shpwt", 0);
+ err = tsleep(&shep_chan, PWAIT | PCATCH, "shpwt", 0);
+ if (err)
+ continue;
+// mtx_unlock(&shep_mtx);
+ shep_chan.shep_error = kproc_create(nfssvc_iod,
+ &shep_chan.shep_inst, NULL, RFHIGHPID, 0, "nfsiod %d",
+ shep_chan.shep_niod);
+ shep_chan.shep_done = 1;
+// mtx_lock(&shep_mtx);
+ }
+// mtx_unlock(&shep_mtx);
+}
+SYSINIT(shepherd, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, shep_thread, NULL);
+
int
nfs_nfsiodnew(int set_iodwant)
{
- int error, i;
+ int i;
int newiod;
if (nfs_numasync >= nfs_iodmax)
@@ -178,11 +207,16 @@
return (-1);
if (set_iodwant > 0)
nfs_iodwant[i] = NFSIOD_CREATED_FOR_NFS_ASYNCIO;
+ bzero(&shep_chan, sizeof(shep_chan));
+ shep_chan.shep_inst = nfs_asyncdaemon + i;
+ shep_chan.shep_niod = newiod;
mtx_unlock(&nfs_iod_mtx);
- error = kproc_create(nfssvc_iod, nfs_asyncdaemon + i, NULL, RFHIGHPID,
- 0, "nfsiod %d", newiod);
+ wakeup(&shep_chan);
mtx_lock(&nfs_iod_mtx);
- if (error) {
+ while (shep_chan.shep_done)
+ continue;
+ shep_chan.shep_done = 0;
+ if (shep_chan.shep_error) {
if (set_iodwant > 0)
nfs_iodwant[i] = NFSIOD_NOT_AVAILABLE;
return (-1);
Index: sys/nfsclient/nfs.h
===================================================================
--- sys/nfsclient/nfs.h (revision 211279)
+++ sys/nfsclient/nfs.h (working copy)
@@ -125,6 +125,7 @@
extern struct nfsstats nfsstats;
extern struct mtx nfs_iod_mtx;
+extern struct mtx shep_mtx;
extern int nfs_numasync;
extern unsigned int nfs_iodmax;
help
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?AANLkTimfZ=UCiX9eo9XCwcROwViFCxJoB-rD1JxdAt2B>