Date: Sat, 24 Aug 2013 16:36:43 +0200 From: Martin Laabs <mailinglists@martinlaabs.de> To: freebsd-arm@freebsd.org Subject: ARM Deadlock - help debugging Message-ID: <5218C4FB.5050709@martinlaabs.de>
next in thread | raw e-mail | index | archive | help
Hi,
I think the problem with the hanging arm is a deadlock anywhere in the file
system.
Therefore I added the following into the kernel config file:
options INVARIANTS
options INVARIANT_SUPPORT
options WITNESS
options DEBUG_LOCKS
options DEBUG_VFS_LOCKS
options DIAGNOSTIC
>From time to time I get the following output for dmesg:
lock order reversal:
1st 0xc2fa0b74 ufs (ufs) @
/usr/home/martin/Rasperry/head/sys/kern/vfs_lookup.c
:518
2nd 0xcc4f00d8 bufwait (bufwait) @
/usr/home/martin/Rasperry/head/sys/ufs/ffs/f
fs_vnops.c:262
3rd 0xc30f28a4 ufs (ufs) @
/usr/home/martin/Rasperry/head/sys/kern/vfs_subr.c:2
099
KDB: stack backtrace:
db_trace_self() at db_trace_self
pc = 0xc0474e34 lr = 0xc012e444 (db_trace_self_wrapper+0x30)
sp = 0xdc9e6410 fp = 0xdc9e6528
r10 = 0xc04f1881
db_trace_self_wrapper() at db_trace_self_wrapper+0x30
pc = 0xc012e444 lr = 0xc0276910 (kdb_backtrace+0x38)
sp = 0xdc9e6530 fp = 0xdc9e6538
r4 = 0xc05c0964 r5 = 0xc04d4338
r6 = 0xc04dc976 r7 = 0xc04f1884
kdb_backtrace() at kdb_backtrace+0x38
pc = 0xc0276910 lr = 0xc0290e14 (witness_checkorder+0xddc)
sp = 0xdc9e6540 fp = 0xdc9e6590
r4 = 0xc04c3234
witness_checkorder() at witness_checkorder+0xddc
pc = 0xc0290e14 lr = 0xc0225424 (__lockmgr_args+0x698)
sp = 0xdc9e6598 fp = 0xdc9e6600
r4 = 0x00080100 r5 = 0x00000833
r6 = 0xc30f28a4 r7 = 0x00080000
r8 = 0x00000100 r9 = 0xc30f2910
r10 = 0xc04dc973
__lockmgr_args() at __lockmgr_args+0x698
pc = 0xc0225424 lr = 0xc042de1c (ffs_lock+0x90)
sp = 0xdc9e6608 fp = 0xdc9e6638
r4 = 0xdc9e6658 r5 = 0x00080100
r6 = 0xc30f2870 r7 = 0xc30f28a4
r8 = 0xc04f1af3 r9 = 0xc30f2910
r10 = 0x00000000
ffs_lock() at ffs_lock+0x90
pc = 0xc042de1c lr = 0xc0495ac0 (VOP_LOCK1_APV+0xd8)
sp = 0xdc9e6640 fp = 0xdc9e6650
r4 = 0xdc9e6658 r5 = 0xc0594770
r6 = 0x00000000 r7 = 0x00080100
r8 = 0xc2f7f960 r9 = 0x00000833
r10 = 0xc04dc973
VOP_LOCK1_APV() at VOP_LOCK1_APV+0xd8
pc = 0xc0495ac0 lr = 0xc02f1d80 (_vn_lock+0x94)
sp = 0xdc9e6658 fp = 0xdc9e6688
r4 = 0xc30f2870 r5 = 0x00000000
r6 = 0x00080100
_vn_lock() at _vn_lock+0x94
pc = 0xc02f1d80 lr = 0xc02e1b48 (vget+0x60)
sp = 0xdc9e6690 fp = 0xdc9e66b0
r4 = 0xc30f2870 r5 = 0x00000000
r6 = 0x00080100 r7 = 0x00000000
r8 = 0xc2f7f960 r9 = 0xc04dbb75
r10 = 0x00000000
vget() at vget+0x60
pc = 0xc02e1b48 lr = 0xc02d6000 (vfs_hash_get+0xe4)
sp = 0xdc9e66b8 fp = 0xdc9e66e8
r4 = 0xc2e0e2f8 r5 = 0x00000000
r6 = 0x0001d59d r7 = 0x00000000
r8 = 0xc30f2870
vfs_hash_get() at vfs_hash_get+0xe4
pc = 0xc02d6000 lr = 0xc0428e60 (ffs_vgetf+0x3c)
sp = 0xdc9e66f0 fp = 0xdc9e6740
r4 = 0x0001d59d r5 = 0xc04ee444
r6 = 0xc30fd140 r7 = 0xdc9e67a8
r8 = 0xc30fd140 r9 = 0x00080000
r10 = 0xc2e0e2f8
ffs_vgetf() at ffs_vgetf+0x3c
pc = 0xc0428e60 lr = 0xc0421ac4 (softdep_sync_buf+0x974)
sp = 0xdc9e6748 fp = 0xdc9e67c8
r4 = 0x0001d59d r5 = 0xc04ee444
r6 = 0xc30fd140 r7 = 0xc04ee444
r8 = 0xc30fd140 r9 = 0x0000088b
r10 = 0x0001d59d
softdep_sync_buf() at softdep_sync_buf+0x974
pc = 0xc0421ac4 lr = 0xc042ec04 (ffs_syncvnode+0x2bc)
sp = 0xdc9e67d0 fp = 0xdc9e6820
r4 = 0x00000000 r5 = 0x00000000
r6 = 0x00000400 r7 = 0xc2fa0b40
r8 = 0xc04f1881 r9 = 0xcc4f00d8
r10 = 0xcc4f0080
ffs_syncvnode() at ffs_syncvnode+0x2bc
pc = 0xc042ec04 lr = 0xc0403f64 (ffs_truncate+0x790)
sp = 0xdc9e6828 fp = 0xdc9e69d8
r4 = 0x00000000 r5 = 0xc2fa0b40
r6 = 0x00000200 r7 = 0xc2e17000
r8 = 0x00000000 r9 = 0x00000000
r10 = 0xc2fa0b40
ffs_truncate() at ffs_truncate+0x790
pc = 0xc0403f64 lr = 0xc0436414 (ufs_direnter+0x8a8)
sp = 0xdc9e69e0 fp = 0xdc9e6a48
r4 = 0xc2fa0b40 r5 = 0x00000000
r6 = 0xc30f2870 r7 = 0xc2f9e700
r8 = 0x00000014 r9 = 0xc2fa0b40
r10 = 0xcd218134
ufs_direnter() at ufs_direnter+0x8a8
pc = 0xc0436414 lr = 0xc043f6d0 (ufs_makeinode+0x3f4)
sp = 0xdc9e6a50 fp = 0xdc9e6bb0
r4 = 0xdc9e6a70 r5 = 0xc2f9d300
r6 = 0xdc9e6d40 r7 = 0x00000000
r8 = 0xc2fa0b40 r9 = 0xdc9e6d28
r10 = 0x000081a4
ufs_makeinode() at ufs_makeinode+0x3f4
pc = 0xc043f6d0 lr = 0xc043bb34 (ufs_create+0x24)
sp = 0xdc9e6bb8 fp = 0xdc9e6bb8
r4 = 0xdc9e6c88 r5 = 0xc0594c90
r6 = 0x00000000 r7 = 0x00000602
r8 = 0x00000000 r9 = 0xdc9e6d28
r10 = 0xdc9e6d48
ufs_create() at ufs_create+0x24
pc = 0xc043bb34 lr = 0xc04934d4 (VOP_CREATE_APV+0xec)
sp = 0xdc9e6bc0 fp = 0xdc9e6bd0
VOP_CREATE_APV() at VOP_CREATE_APV+0xec
pc = 0xc04934d4 lr = 0xc02f1640 (vn_open_cred+0x268)
sp = 0xdc9e6bd8 fp = 0xdc9e6cb8
r4 = 0xdc9e6ce8 r5 = 0xdc9e6d28
r6 = 0xc2fa0b40
vn_open_cred() at vn_open_cred+0x268
pc = 0xc02f1640 lr = 0xc02f13d0 (vn_open+0x24)
sp = 0xdc9e6cc0 fp = 0xdc9e6cc8
r4 = 0xc2f7f960 r5 = 0xdc9e6d10
r6 = 0xc2decc00 r7 = 0xdc9e6ce8
r8 = 0x00000012 r9 = 0x00000000
r10 = 0x00000000
vn_open() at vn_open+0x24
pc = 0xc02f13d0 lr = 0xc02ea9f8 (kern_openat+0x1a4)
sp = 0xdc9e6cd0 fp = 0xdc9e6da8
kern_openat() at kern_openat+0x1a4
pc = 0xc02ea9f8 lr = 0xc02ea7e8 (sys_open+0x28)
sp = 0xdc9e6db0 fp = 0xdc9e6db8
r4 = 0xc2f7f960 r5 = 0x00000000
r6 = 0x2092e2ec r7 = 0x00000000
r8 = 0xdc9e6e10 r9 = 0xc2df6320
r10 = 0x00000001
sys_open() at sys_open+0x28
pc = 0xc02ea7e8 lr = 0xc0484c44 (swi_handler+0x284)
sp = 0xdc9e6dc0 fp = 0xdc9e6e58
swi_handler() at swi_handler+0x284
pc = 0xc0484c44 lr = 0xc0476490 (swi_entry+0x2c)
sp = 0xdc9e6e60 fp = 0xbfffe310
r4 = 0x2091a330 r5 = 0x2083d138
r6 = 0x2092e2ec r7 = 0x00000005
r8 = 0x00000001 r9 = 0x00000000
swi_entry() at swi_entry+0x2c
pc = 0xc0476490 lr = 0xc0476490 (swi_entry+0x2c)
sp = 0xdc9e6e60 fp = 0xbfffe310
Since I am not familiar with the FreeBSD kernel internals I would like to
know if this is something I should try to investigate further.
Thank you,
Martin
PS: Is it possible to display the dmesg continuously like you can have a
look to logfiles with tail -f?
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?5218C4FB.5050709>