Date: Mon, 4 Feb 2013 16:20:36 +0300 From: Sergey Kandaurov <pluknet@gmail.com> To: Andriy Gapon <avg@freebsd.org> Cc: Konstantin Belousov <kostikbel@gmail.com>, Rick Macklem <rmacklem@uoguelph.ca>, FreeBSD Current <freebsd-current@freebsd.org> Subject: Re: panic: LK_RETRY set with incompatible flags Message-ID: <CAE-mSOJjtHqqXzgXyUw%2Bn0nEWK1MJVuu6ETZFu4Ox5FLdY-8nQ@mail.gmail.com> In-Reply-To: <CAE-mSOLfe%2B=JpVCB0TGdJu3cOz-3f-WP93dQH4YWu4ZTxvt-9Q@mail.gmail.com> References: <510E9877.5000701@FreeBSD.org> <1515954355.2640466.1359940065810.JavaMail.root@erie.cs.uoguelph.ca> <CAE-mSO%2BJMk=SuYr7=g6MdT_=44c7%2BB00FF2YBaiUkJJaTgVi3Q@mail.gmail.com> <510FA448.8020204@FreeBSD.org> <CAE-mSOJdgy4PvVu4Hp5=x=AuntEmzJU6Dr2VAuy_4evm3-6ozQ@mail.gmail.com> <CAE-mSOLfe%2B=JpVCB0TGdJu3cOz-3f-WP93dQH4YWu4ZTxvt-9Q@mail.gmail.com>
next in thread | previous in thread | raw e-mail | index | archive | help
On 4 February 2013 16:30, Sergey Kandaurov <pluknet@gmail.com> wrote:
> On 4 February 2013 16:13, Sergey Kandaurov <pluknet@gmail.com> wrote:
>> On 4 February 2013 16:06, Andriy Gapon <avg@freebsd.org> wrote:
>>> on 04/02/2013 13:49 Sergey Kandaurov said the following:
>>>> Hi, Rick! Here is the requested info regarding witness, and a bit more.
>>>> The triggered KASSERT is now different though.
>>>
>>> It's exactly the same problem though :-)
>>> Do you have a crashdump?
>>> If yes, please print **vpp.
>>
>> Yep, but it's in a bad form :(
>> It has many bits optimized out (compiled with clang).
>> I'll rebuild the kernel with -O or so and will try again.
>>
>> #8 0xffffffff808bc4ce in kdb_enter (why=0xffffffff80e7ed99 "panic",
>> msg=<value optimized out>) at cpufunc.h:63
>> #9 0xffffffff80888fb7 in vpanic (fmt=<value optimized out>,
>> ap=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:746
>> #10 0xffffffff80888e66 in kassert_panic (fmt=<value optimized out>)
>> at /usr/src/sys/kern/kern_shutdown.c:641
>> #11 0xffffffff808d2259 in witness_checkorder (lock=0xfffffe00a1b44240,
>> ---Type <return> to continue, or q <return> to quit---
>> flags=1,
>> file=0xffffffff81b2bb36
>> "/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c",
>> line=1452,
>> interlock=<value optimized out>) at /usr/src/sys/kern/subr_witness.c:1122
>> #12 0xffffffff8086c11e in __lockmgr_args (lk=0xfffffe00a1b44240,
>> flags=<value optimized out>, ilk=0xfffffe00a1b44270,
>> wmesg=0xffffffff81b2808d "zfs", pri=96, timo=51,
>> file=0xffffffff80e8e407 "/usr/src/sys/kern/vfs_default.c", line=0)
>> at /usr/src/sys/kern/kern_lock.c:511
>> #13 0xffffffff8091439c in vop_stdlock (ap=<value optimized out>)
>> at lockmgr.h:97
>> #14 0xffffffff80cb70c0 in VOP_LOCK1_APV (vop=<value optimized out>,
>> a=<value optimized out>) at vnode_if.c:2022
>> #15 0xffffffff80932fbb in _vn_lock (vp=0xfffffe00a1b441d8,
>> flags=<value optimized out>,
>> file=0xffffffff81b2bb36
>> "/usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c",
>> line=1452) at vnode_if.h:859
>> #16 0xffffffff81abd902 in zfs_lookup ()
>> at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:1452
>> #17 0xffffffff81abdc1d in zfs_freebsd_lookup (ap=0xffffff848e6c0270)
>> at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:5864
>> ---Type <return> to continue, or q <return> to quit---
>> #18 0xffffffff80cb51b2 in VOP_CACHEDLOOKUP_APV (vop=<value optimized out>,
>> a=<value optimized out>) at vnode_if.c:191
>> #19 0xffffffff8091177f in vfs_cache_lookup (ap=<value optimized out>)
>> at vnode_if.h:80
>> #20 0xffffffff80cb50a2 in VOP_LOOKUP_APV (vop=<value optimized out>,
>> a=<value optimized out>) at vnode_if.c:125
>> #21 0xffffffff80919658 in lookup (ndp=0xffffff848e6c05d8) at vnode_if.h:54
>> #22 0xffffffff807dffe5 in nfsvno_namei (nd=0xffffff848e6c08c8,
>> ndp=0xffffff848e6c05d8, dp=<value optimized out>,
>> islocked=<value optimized out>, exp=<value optimized out>,
>> p=<value optimized out>, retdirp=<value optimized out>)
>> at /usr/src/sys/fs/nfsserver/nfs_nfsdport.c:413
>> #23 0xffffffff807d8ffa in nfsrvd_lookup (nd=0xffffff848e6c08c8,
>> isdgram=<value optimized out>, dp=0xfffffe00a1b441d8,
>> vpp=0xffffff848e6c0810, fhp=0xffffff848e6c07f4, p=0xfffffe00a1198000,
>> exp=0xffffff848e6c07a0) at /usr/src/sys/fs/nfsserver/nfs_nfsdserv.c:517
>> #24 0xffffffff807cb845 in nfsrvd_dorpc (nd=0xffffff848e6c08c8, isdgram=0,
>> p=0xfffffe00a1198000) at /usr/src/sys/fs/nfsserver/nfs_nfsdsocket.c:823
>> #25 0xffffffff807d7af2 in nfssvc_program (rqst=0xfffffe00a17bb000,
>> xprt=0xfffffe00a124b200) at /usr/src/sys/fs/nfsserver/nfs_nfsdkrpc.c:347
>> #26 0xffffffff80a70659 in svc_run_internal (pool=0xfffffe00067db600,
>> ismaster=0) at /usr/src/sys/rpc/svc.c:895
>> #27 0xffffffff80a7155b in svc_thread_start (arg=0xffffff848e6bfc50)
>> ---Type <return> to continue, or q <return> to quit---
>> at /usr/src/sys/rpc/svc.c:1200
>> #28 0xffffffff80858944 in fork_exit (
>> callout=0xffffffff80a71550 <svc_thread_start>, arg=0xfffffe00067db600,
>> frame=0xffffff848e6c0c00) at /usr/src/sys/kern/kern_fork.c:991
>> #29 0xffffffff80bfa86e in fork_trampoline () at exception.S:602
>> #30 0x0000000000000080 in ?? ()
>> #31 0x00007fffffffd820 in ?? ()
>> #32 0x0000000000000001 in ?? ()
>> #33 0x0000000000000000 in ?? ()
>> Current language: auto; currently minimal
>>
>> (kgdb) p vpp
>> Cannot access memory at address 0x0
>> (kgdb) p dvp
>> Cannot access memory at address 0x0
>>
>
> That was from f 16. And as Andriy suggested in private (thanks Andriy!)
> those pointers could be reached from ap in f 17.
> On the other hand, zfs_lookup() is large, and *vpp might be
> rewritten several times there. Here it is:
>
> (kgdb) f 17
> #17 0xffffffff81abdc1d in zfs_freebsd_lookup (ap=0xffffff848e6c0270)
> at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c:5864
> 5864 return (zfs_lookup(ap->a_dvp, nm, ap->a_vpp, cnp,
> cnp->cn_nameiop,
> (kgdb) p ap->a_vpp
> $4 = (struct vnode **) 0xffffff848e6c0618
> (kgdb) p *ap->a_vpp
> $5 = (struct vnode *) 0xfffffe00a1b441d8
> (kgdb) p **ap->a_vpp
> $6 = {v_tag = 0xffffffff81b2808d "zfs", v_op = 0xffffffff81b35808,
> v_data = 0xfffffe00a188f000, v_mount = 0xfffffe0015a5f000, v_nmntvnodes = {
> tqe_next = 0xfffffe00a1b44000, tqe_prev = 0xfffffe00a1b443d0}, v_un = {
> vu_mount = 0x0, vu_socket = 0x0, vu_cdev = 0x0, vu_fifoinfo = 0x0},
> v_hashlist = {le_next = 0x0, le_prev = 0x0}, v_cache_src = {lh_first = 0x0},
> v_cache_dst = {tqh_first = 0x0, tqh_last = 0xfffffe00a1b44228},
> v_cache_dd = 0x0, v_lock = {lock_object = {
> lo_name = 0xffffffff81b2808d "zfs", lo_flags = 91947008, lo_data = 0,
> lo_witness = 0xffffff80006d2700}, lk_lock = 18446741877389099008,
> lk_exslpfail = 0, lk_timo = 51, lk_pri = 96}, v_interlock = {
> lock_object = {lo_name = 0xffffffff80e873f0 "vnode interlock",
> lo_flags = 16973824, lo_data = 0, lo_witness = 0xffffff80006c9600},
> mtx_lock = 4}, v_vnlock = 0xfffffe00a1b44240, v_actfreelist = {
> tqe_next = 0xfffffe000bd363b0, tqe_prev = 0xfffffe0015a5f078}, v_bufobj = {
> bo_mtx = {lock_object = {lo_name = 0xffffffff80e8f3d5 "bufobj interlock",
> lo_flags = 16973824, lo_data = 0, lo_witness = 0xffffff80006d0580},
> mtx_lock = 4}, bo_ops = 0xffffffff8121cc30,
> bo_object = 0xfffffe00a1872780, bo_synclist = {le_next = 0x0,
> le_prev = 0x0}, bo_private = 0xfffffe00a1b441d8,
> __bo_vnode = 0xfffffe00a1b441d8, bo_clean = {bv_hd = {tqh_first = 0x0,
> tqh_last = 0xfffffe00a1b442f8}, bv_root = 0x0, bv_cnt = 0},
> bo_dirty = {bv_hd = {tqh_first = 0x0, tqh_last = 0xfffffe00a1b44318},
> bv_root = 0x0, bv_cnt = 0}, bo_numoutput = 0, bo_flag = 0,
> ---Type <return> to continue, or q <return> to quit---
> bo_bsize = 131072}, v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0,
> v_rl = {rl_waiters = {tqh_first = 0x0, tqh_last = 0xfffffe00a1b44360},
> rl_currdep = 0x0}, v_cstart = 0, v_lasta = 0, v_lastw = 0, v_clen = 0,
> v_holdcnt = 4, v_usecount = 4, v_iflag = 512, v_vflag = 0, v_writecount = 0,
> v_hash = 10597441, v_type = VDIR}
And here is dvp, cnp. So yes, *vpp == dvp.
(kgdb) p *ap->a_vpp
$7 = (struct vnode *) 0xfffffe00a1b441d8
(kgdb) p ap->a_dvp
$5 = (struct vnode *) 0xfffffe00a1b441d8
(kgdb) p *ap->a_dvp
$8 = {v_tag = 0xffffffff81b2808d "zfs", v_op = 0xffffffff81b35808,
v_data = 0xfffffe00a188f000, v_mount = 0xfffffe0015a5f000,
v_nmntvnodes = {tqe_next = 0xfffffe00a1b44000, tqe_prev =
0xfffffe00a1b443d0}, v_un = {vu_mount = 0x0, vu_socket = 0x0,
vu_cdev = 0x0, vu_fifoinfo = 0x0}, v_hashlist = {le_next = 0x0,
le_prev = 0x0}, v_cache_src = {lh_first = 0x0}, v_cache_dst = {
tqh_first = 0x0, tqh_last = 0xfffffe00a1b44228}, v_cache_dd = 0x0,
v_lock = {lock_object = {lo_name = 0xffffffff81b2808d "zfs",
lo_flags = 91947008, lo_data = 0, lo_witness =
0xffffff80006d2700}, lk_lock = 18446741877389099008, lk_exslpfail = 0,
lk_timo = 51, lk_pri = 96}, v_interlock = {lock_object = {lo_name
= 0xffffffff80e873f0 "vnode interlock", lo_flags = 16973824,
lo_data = 0, lo_witness = 0xffffff80006c9600}, mtx_lock = 4},
v_vnlock = 0xfffffe00a1b44240, v_actfreelist = {
tqe_next = 0xfffffe000bd363b0, tqe_prev = 0xfffffe0015a5f078},
v_bufobj = {bo_mtx = {lock_object = {
lo_name = 0xffffffff80e8f3d5 "bufobj interlock", lo_flags =
16973824, lo_data = 0, lo_witness = 0xffffff80006d0580},
mtx_lock = 4}, bo_ops = 0xffffffff8121cc30, bo_object =
0xfffffe00a1872780, bo_synclist = {le_next = 0x0, le_prev = 0x0},
bo_private = 0xfffffe00a1b441d8, __bo_vnode = 0xfffffe00a1b441d8,
bo_clean = {bv_hd = {tqh_first = 0x0,
tqh_last = 0xfffffe00a1b442f8}, bv_root = 0x0, bv_cnt = 0},
bo_dirty = {bv_hd = {tqh_first = 0x0,
tqh_last = 0xfffffe00a1b44318}, bv_root = 0x0, bv_cnt = 0},
bo_numoutput = 0, bo_flag = 0, bo_bsize = 131072},
v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0, v_rl = {rl_waiters =
{tqh_first = 0x0, tqh_last = 0xfffffe00a1b44360},
rl_currdep = 0x0}, v_cstart = 0, v_lasta = 0, v_lastw = 0, v_clen
= 0, v_holdcnt = 4, v_usecount = 4, v_iflag = 512,
v_vflag = 0, v_writecount = 0, v_hash = 10597441, v_type = VDIR}
(kgdb) p/x *ap->a_dvp
$2 = {v_tag = 0xffffffff81b2808d, v_op = 0xffffffff81b35808, v_data =
0xfffffe00a188f000, v_mount = 0xfffffe0015a5f000,
v_nmntvnodes = {tqe_next = 0xfffffe00a1b44000, tqe_prev =
0xfffffe00a1b443d0}, v_un = {vu_mount = 0x0, vu_socket = 0x0,
vu_cdev = 0x0, vu_fifoinfo = 0x0}, v_hashlist = {le_next = 0x0,
le_prev = 0x0}, v_cache_src = {lh_first = 0x0}, v_cache_dst = {
tqh_first = 0x0, tqh_last = 0xfffffe00a1b44228}, v_cache_dd = 0x0,
v_lock = {lock_object = {lo_name = 0xffffffff81b2808d,
lo_flags = 0x57b0000, lo_data = 0x0, lo_witness =
0xffffff80006d2700}, lk_lock = 0xfffffe00a1198000, lk_exslpfail = 0x0,
lk_timo = 0x33, lk_pri = 0x60}, v_interlock = {lock_object =
{lo_name = 0xffffffff80e873f0, lo_flags = 0x1030000, lo_data = 0x0,
lo_witness = 0xffffff80006c9600}, mtx_lock = 0x4}, v_vnlock =
0xfffffe00a1b44240, v_actfreelist = {
tqe_next = 0xfffffe000bd363b0, tqe_prev = 0xfffffe0015a5f078},
v_bufobj = {bo_mtx = {lock_object = {
lo_name = 0xffffffff80e8f3d5, lo_flags = 0x1030000, lo_data =
0x0, lo_witness = 0xffffff80006d0580}, mtx_lock = 0x4},
bo_ops = 0xffffffff8121cc30, bo_object = 0xfffffe00a1872780,
bo_synclist = {le_next = 0x0, le_prev = 0x0},
bo_private = 0xfffffe00a1b441d8, __bo_vnode = 0xfffffe00a1b441d8,
bo_clean = {bv_hd = {tqh_first = 0x0,
tqh_last = 0xfffffe00a1b442f8}, bv_root = 0x0, bv_cnt = 0x0},
bo_dirty = {bv_hd = {tqh_first = 0x0,
tqh_last = 0xfffffe00a1b44318}, bv_root = 0x0, bv_cnt = 0x0},
bo_numoutput = 0x0, bo_flag = 0x0, bo_bsize = 0x20000},
v_pollinfo = 0x0, v_label = 0x0, v_lockf = 0x0, v_rl = {rl_waiters =
{tqh_first = 0x0, tqh_last = 0xfffffe00a1b44360},
rl_currdep = 0x0}, v_cstart = 0x0, v_lasta = 0x0, v_lastw = 0x0,
v_clen = 0x0, v_holdcnt = 0x4, v_usecount = 0x4,
v_iflag = 0x200, v_vflag = 0x0, v_writecount = 0x0, v_hash =
0xa1b441, v_type = 0x2}
(kgdb) p ap->a_cnp
$9 = (struct componentname *) 0xffffff848e6c0640
(kgdb) p *ap->a_cnp
$10 = {cn_nameiop = 0, cn_flags = 8451076, cn_thread =
0xfffffe00a1198000, cn_cred = 0xfffffe00a167cc00, cn_lkflags = 524288,
cn_pnbuf = 0xfffffe000bdee400 "..", cn_nameptr = 0xfffffe000bdee400
"..", cn_namelen = 2, cn_consume = 0}
(kgdb) p/x *ap->a_cnp
$4 = {cn_nameiop = 0x0, cn_flags = 0x80f404, cn_thread =
0xfffffe00a1198000, cn_cred = 0xfffffe00a167cc00, cn_lkflags =
0x80000,
cn_pnbuf = 0xfffffe000bdee400, cn_nameptr = 0xfffffe000bdee400,
cn_namelen = 0x2, cn_consume = 0x0}
--
wbr,
pluknet
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?CAE-mSOJjtHqqXzgXyUw%2Bn0nEWK1MJVuu6ETZFu4Ox5FLdY-8nQ>