Date: Thu, 29 Dec 2011 13:10:05 -0800 From: David P Discher <dpd@bitgravity.com> To: Johannes Totz <jtotz@imperial.ac.uk> Cc: freebsd-fs@freebsd.org Subject: Re: fusefs broken on 8-stable? Message-ID: <87E5C7D5-2D73-4A42-B871-8B27448B0DD9@bitgravity.com> In-Reply-To: <isc2ui$kq$1@dough.gmane.org> References: <iqj9pf$mrn$1@dough.gmane.org> <isc2ui$kq$1@dough.gmane.org>
next in thread | previous in thread | raw e-mail | index | archive | help
Johannes - Were you able to get a resolution for this ?
I've been able to replicate this panic in fiov_adjust(), but being =
called from vnlru, looks like its doing VOP_RECLAIM_APV().
I'm willing to dive deeper and debug this, however would appreciate some =
guidance form the community. I'm having issues wrapping my head around =
macro FUSE_DIMALLOC(), which I think is using some sort of anonymous =
function call to get to fiov_adjust(). I've been battling this for a =
week or two now, so I have symbols in fuse module, so I have a bit more =
info.
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
(kgdb) up
#11 0xffffffff81147412 in fdisp_make_pid =
(fdip=3D0xffffff9bce6399f0, mp=3DVariable "mp" is not available.
)
at fuse_msg.c:788
788 FUSE_DIMALLOC(&fdip->tick->tk_ms_fiov, =
fdip->finh,
(kgdb) l
783 if (fdip->tick)
784 fticket_refresh(fdip->tick);
785 else
786 fdip->tick =3D fuse_ticket_fetch(data);
787
788 FUSE_DIMALLOC(&fdip->tick->tk_ms_fiov, =
fdip->finh,
789 fdip->indata, fdip->iosize);
790 fuse_setup_ihead(fdip->finh, fdip->tick, nid, =
op, fdip->iosize, pid, cred);
791 }
792
(kgdb) p fdip
$4 =3D (struct fuse_dispatcher *) 0xffffff9bce6399f0
(kgdb) p *(struct fuse_dispatcher *)fdip
$5 =3D {tick =3D 0x0, finh =3D 0x0, indata =3D =
0xffffff9bce639a30, iosize =3D 8,=20
nodeid =3D 0, answ_stat =3D 1356898584, answ =3D =
0xffffff0350e0a138}
(kgdb) p *(struct fuse_dispatcher *)fdip->tick
Cannot access memory at address 0x0
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
(kgdb) up
#9 0xffffffff807bd7b8 in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:228
228 call trap
Current language: auto; currently asm
(kgdb) up
#10 0xffffffff81146b17 in fiov_adjust (fiov=3D0x30, size=3D48) =
at fuse_msg.c:96
96 {
Current language: auto; currently c
(kgdb) l
91 free(fiov->base, M_FUSEMSG);
92 }
93
94 void
95 fiov_adjust(struct fuse_iov *fiov, size_t size)
96 {
97 if (fiov->allocated_size < size ||
98 (fuse_iov_permanent_bufsize >=3D 0 &&
99 fiov->allocated_size - size > =
fuse_iov_permanent_bufsize &&
100 --fiov->credit < 0)) {
(kgdb) p fiov
$1 =3D (struct fuse_iov *) 0x30
(kgdb) p (struct fuse_iov *)fiov
$2 =3D (struct fuse_iov *) 0x30
(kgdb) p *(struct fuse_iov *)fiov
Cannot access memory at address 0x30
(kgdb) p size
$3 =3D 48
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
(kgdb) bt
#0 doadump () at /usr/src/sys/kern/kern_shutdown.c:263
#1 0xffffffff801f6c9c in db_fncall (dummy1=3DVariable "dummy1" =
is not available.
)
at /usr/src/sys/ddb/db_command.c:548
#2 0xffffffff801f6fd1 in db_command =
(last_cmdp=3D0xffffffff80b67a40, cmd_table=3DVariable "cmd_table" is not =
available.
)
at /usr/src/sys/ddb/db_command.c:445
#3 0xffffffff801f7220 in db_command_loop ()
at /usr/src/sys/ddb/db_command.c:498
#4 0xffffffff801f9269 in db_trap (type=3DVariable "type" is not =
available.
) at /usr/src/sys/ddb/db_main.c:229
#5 0xffffffff80575ff1 in kdb_trap (type=3D12, code=3D0, =
tf=3D0xffffff9bce6398a0)
at /usr/src/sys/kern/subr_kdb.c:548
#6 0xffffffff807d4bad in trap_fatal (frame=3D0xffffff9bce6398a0, =
eva=3DVariable "eva" is not available.
)
at /usr/src/sys/amd64/amd64/trap.c:820
#7 0xffffffff807d4eea in trap_pfault (frame=3D0xffffff9bce6398a0,=
usermode=3D0)
at /usr/src/sys/amd64/amd64/trap.c:741
#8 0xffffffff807d52ff in trap (frame=3D0xffffff9bce6398a0)
at /usr/src/sys/amd64/amd64/trap.c:478
#9 0xffffffff807bd7b8 in calltrap ()
at /usr/src/sys/amd64/amd64/exception.S:228
#10 0xffffffff81146b17 in fiov_adjust (fiov=3D0x30, size=3D48) =
at fuse_msg.c:96
#11 0xffffffff81147412 in fdisp_make_pid =
(fdip=3D0xffffff9bce6399f0, mp=3DVariable "mp" is not available.
)
at fuse_msg.c:788
#12 0xffffffff8114a6d4 in fuse_send_forget (mp=3DVariable "mp" =
is not available.
) at fuse_vnops.c:378
#13 0xffffffff8114a7d6 in fuse_recyc_backend =
(vp=3D0xffffff0350e0a000,=20
td=3D0xffffff0029533000) at fuse_vnops.c:539
#14 0xffffffff808272fa in VOP_RECLAIM_APV =
(vop=3D0xffffffff811504e0,=20
a=3D0xffffff9bce639a90) at vnode_if.c:1926
#15 0xffffffff805ccf95 in vgonel (vp=3D0xffffff0350e0a000) at =
vnode_if.h:830
#16 0xffffffff805cf9c4 in vnlru_free (count=3D28)
at /usr/src/sys/kern/vfs_subr.c:975
#17 0xffffffff805d030f in vnlru_proc () at =
/usr/src/sys/kern/vfs_subr.c:834
#18 0xffffffff805211bd in fork_exit (callout=3D0xffffffff805cfec0 =
<vnlru_proc>,=20
arg=3D0x0, frame=3D0xffffff9bce639c50) at =
/usr/src/sys/kern/kern_fork.c:876
#19 0xffffffff807bdcfe in fork_trampoline ()
at /usr/src/sys/amd64/amd64/exception.S:602
=09
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
Fatal trap 12: page fault while in kernel mode^M
cpuid =3D 7; apic id =3D 15^M
fault virtual address =3D 0x40^M
fault code =3D supervisor read data, page not =
present^M
instruction pointer =3D 0x20:0xffffffff81146b17^M
stack pointer =3D 0x28:0xffffff9bce639950^M
frame pointer =3D 0x28:0xffffff9bce639980^M
code segment =3D base 0x0, limit 0xfffff, type 0x1b^M
=3D DPL 0, pres =
1, long 1, def32 0, gran 1^M
processor eflags =3D interrupt enabled, resume, IOPL =3D =
0^M
current process =3D 18 (vnlru)^M
[thread pid 18 tid 100138 ]
Stopped at fiov_adjust+0x17: movq 0x10(%rdi),%rax
db> [-- MARK -- Thu Dec 29 01:15:00 2011]
bt
Tracing pid 18 tid 100138 td 0xffffff0029533000
fiov_adjust() at fiov_adjust+0x17
fdisp_make_pid() at fdisp_make_pid+0xd2
fuse_send_forget() at fuse_send_forget+0x44
fuse_recyc_backend() at fuse_recyc_backend+0xc6
VOP_RECLAIM_APV() at VOP_RECLAIM_APV+0x4a
vgonel() at vgonel+0xf5
vnlru_free() at vnlru_free+0x294
vnlru_proc() at vnlru_proc+0x44f
fork_exit() at fork_exit+0x11d
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff9bce639d00, rbp =3D 0 ---
db> call doadump=20
=20
---
David P. Discher
dpd@bitgravity.com * AIM: bgDavidDPD
BITGRAVITY * http://www.bitgravity.com
On Jun 3, 2011, at 6:49 PM, Johannes Totz wrote:
> On 13/05/2011 13:56, Johannes Totz wrote:
>> Heya!
>>=20
>> Using encfs (built on top of fuse) gives me panics in combination =
with
>> rsync. Dump didn't succeed. The info below is transcribbled from a
>> photograph. This is repeatable.
>> Without dump this is probably not very helpful....
>>=20
>>=20
>> # uname -a
>> FreeBSD XXX 8.2-STABLE FreeBSD 8.2-STABLE #1: Thu Mar 10 23:30:08 GMT
>> 2011 root@XXX:/usr/obj/usr/src/sys/GENERIC amd64
>=20
> Managed to get a dump:
>=20
> Fatal trap 12: page fault while in kernel mode
> cpuid =3D 1; apic id =3D 01
> fault virtual address =3D 0x40
> fault code =3D supervisor read data, page not present
> instruction pointer =3D 0x20:0xffffffff81f2ba3c
> stack pointer =3D 0x28:0xffffff813aa2f570
> frame pointer =3D 0x28:0xffffff813aa2f5a0
> code segment =3D base 0x0, limit 0xfffff, type 0x1b
> =3D DPL 0, pres 1, long 1, def32 0, gran 1
> processor eflags =3D interrupt enabled, resume, IOPL =3D 0
> current process =3D 1560 (rsync)
> trap number =3D 12
> panic: page fault
> cpuid =3D 1
> KDB: stack backtrace:
> #0 0xffffffff805cc226 at kdb_backtrace+0x5c
> #1 0xffffffff8059d000 at panic+0x1b4
> #2 0xffffffff80875a75 at trap_fatal+0x394
> #3 0xffffffff80875ce5 at trap_pfault+0x252
> #4 0xffffffff808761b5 at trap+0x3f4
> #5 0xffffffff8085f384 at calltrap+0x8
> #6 0xffffffff81f2c170 at fdisp_make+0xe4
> #7 0xffffffff81f3033d at fuse_lookup+0x1dc
> #8 0xffffffff808caac8 at VOP_LOOKUP_APV+0x4c
> #9 0xffffffff8061b0db at lookup+0x61e
> #10 0xffffffff8061bf1f at namei+0x592
> #11 0xffffffff80629df4 at kern_statat_vnhook+0x87
> #12 0xffffffff80629f98 at kern_statat+0x15
> #13 0xffffffff80629fb6 at kern_lstat+0x1c
> #14 0xffffffff8062a03d at lstat+0x25
> #15 0xffffffff805d7cf4 at syscallenter+0x2d9
> #16 0xffffffff80875d45 at syscall+0x38
> #17 0xffffffff8085f662 at Xfast_syscall+0xe2
>=20
>=20
> #0 doadump () at pcpu.h:224
> #1 0xffffffff8059cd12 in boot (howto=3D260) at
> /usr/src/sys/kern/kern_shutdown.c:419
> #2 0xffffffff8059d03b in panic (fmt=3DVariable "fmt" is not =
available.)
> at /usr/src/sys/kern/kern_shutdown.c:592
> #3 0xffffffff80875a75 in trap_fatal (frame=3D0xffffff813aa2f4c0, =
eva=3D64)
> at /usr/src/sys/amd64/amd64/trap.c:809
> #4 0xffffffff80875ce5 in trap_pfault (frame=3D0xffffff813aa2f4c0,
> usermode=3D0) at /usr/src/sys/amd64/amd64/trap.c:725
> #5 0xffffffff808761b5 in trap (frame=3D0xffffff813aa2f4c0) at
> /usr/src/sys/amd64/amd64/trap.c:475
> #6 0xffffffff8085f384 in calltrap () at
> /usr/src/sys/amd64/amd64/exception.S:228
> #7 0xffffffff81f2ba3c in fiov_adjust () from =
/usr/local/modules/fuse.ko
> #8 0xffffffff81f2c170 in fdisp_make () from =
/usr/local/modules/fuse.ko
> #9 0xffffffff81f3033d in fuse_lookup () from =
/usr/local/modules/fuse.ko
> #10 0xffffffff808caac8 in VOP_LOOKUP_APV (vop=3D0x0, a=3D0x30) at =
vnode_if.c:123
> #11 0xffffffff8061b0db in lookup (ndp=3D0x618) at vnode_if.h:54
> #12 0xffffffff8061bf1f in namei (ndp=3D0xffffff813aa2f8e0) at
> /usr/src/sys/kern/vfs_lookup.c:269
> #13 0xffffffff80629df4 in kern_statat_vnhook (td=3D0xffffff00039f2460,
> flag=3DVariable "flag" is not available.) at
> /usr/src/sys/kern/vfs_syscalls.c:2346
> #14 0xffffffff80629f98 in kern_statat (td=3DVariable "td" is not
> available.) at /usr/src/sys/kern/vfs_syscalls.c:2327
> #15 0xffffffff80629fb6 in kern_lstat (td=3DVariable "td" is not
> available.) at /usr/src/sys/kern/vfs_syscalls.c:2400
> #16 0xffffffff8062a03d in lstat (td=3DVariable "td" is not available.) =
at
> /usr/src/sys/kern/vfs_syscalls.c:2390
> #17 0xffffffff805d7cf4 in syscallenter (td=3D0xffffff00039f2460,
> sa=3D0xffffff813aa2fbc0) at /usr/src/sys/kern/subr_trap.c:315
> #18 0xffffffff80875d45 in syscall (frame=3D0xffffff813aa2fc50) at
> /usr/src/sys/amd64/amd64/trap.c:914
> #19 0xffffffff8085f662 in Xfast_syscall () at
> /usr/src/sys/amd64/amd64/exception.S:381
> #20 0x000000080097a5dc in ?? ()
>=20
> At #10 a=3D0x30 looks dodgy and fault address is 0x40, but I have no =
idea
> what that means...
>=20
> _______________________________________________
> freebsd-fs@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-fs
> To unsubscribe, send any mail to "freebsd-fs-unsubscribe@freebsd.org"
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?87E5C7D5-2D73-4A42-B871-8B27448B0DD9>