Date: Mon, 18 Mar 2019 21:11:40 +0000 From: bugzilla-noreply@freebsd.org To: fs@FreeBSD.org Subject: [Bug 235783] Repeated ZFS-related kernel panic Message-ID: <bug-235783-3630-nuDCDTR8Wu@https.bugs.freebsd.org/bugzilla/> In-Reply-To: <bug-235783-3630@https.bugs.freebsd.org/bugzilla/> References: <bug-235783-3630@https.bugs.freebsd.org/bugzilla/>
next in thread | previous in thread | raw e-mail | index | archive | help
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=3D235783 --- Comment #5 from Jurij Kovacic <jurij.kovacic@ocpea.com> --- Hello Andriy, Thank you very much for the explanation.=20 After running: kgdb /boot/kernel/kernel /var/crash/vmcore.last the instruction at "0xffffffff82299013" is: 0xffffffff82299013 <dbuf_destroy+563>: mov (%rax),%rcx Please find the complete disassembly of the dbuf_destroy function below. Kind regards, Jurij Dump of assembler code for function dbuf_destroy: 0xffffffff82298de0 <dbuf_destroy+0>: push %rbp 0xffffffff82298de1 <dbuf_destroy+1>: mov %rsp,%rbp 0xffffffff82298de4 <dbuf_destroy+4>: push %r15 0xffffffff82298de6 <dbuf_destroy+6>: push %r14 0xffffffff82298de8 <dbuf_destroy+8>: push %r13 0xffffffff82298dea <dbuf_destroy+10>: push %r12 0xffffffff82298dec <dbuf_destroy+12>: push %rbx 0xffffffff82298ded <dbuf_destroy+13>: sub $0x18,%rsp 0xffffffff82298df1 <dbuf_destroy+17>: mov %rdi,%r13 0xffffffff82298df4 <dbuf_destroy+20>: mov 0x30(%r13),%r14 0xffffffff82298df8 <dbuf_destroy+24>: mov 0x88(%r13),%rdi 0xffffffff82298dff <dbuf_destroy+31>: test %rdi,%rdi 0xffffffff82298e02 <dbuf_destroy+34>: je 0xffffffff82298e17 <dbuf_destroy+55> 0xffffffff82298e04 <dbuf_destroy+36>: mov %r13,%rsi 0xffffffff82298e07 <dbuf_destroy+39>: callq 0xffffffff8228c220 <arc_buf_destroy> 0xffffffff82298e0c <dbuf_destroy+44>: movq $0x0,0x88(%r13) 0xffffffff82298e17 <dbuf_destroy+55>: cmpq $0xffffffffffffffff,0x40(%r1= 3) 0xffffffff82298e1c <dbuf_destroy+60>: jne 0xffffffff82298e43 <dbuf_destroy+99> 0xffffffff82298e1e <dbuf_destroy+62>: mov 0x18(%r13),%rdi 0xffffffff82298e22 <dbuf_destroy+66>: mov $0x140,%esi 0xffffffff82298e27 <dbuf_destroy+71>: callq 0xffffffff82328d00 <zio_buf_free> 0xffffffff82298e2c <dbuf_destroy+76>: mov $0x140,%edi 0xffffffff82298e31 <dbuf_destroy+81>: mov $0x4,%esi 0xffffffff82298e36 <dbuf_destroy+86>: callq 0xffffffff8228b6c0 <arc_space_return> 0xffffffff82298e3b <dbuf_destroy+91>: movl $0x0,0x78(%r13) 0xffffffff82298e43 <dbuf_destroy+99>: mov 0xd8(%r13),%r15 0xffffffff82298e4a <dbuf_destroy+106>: test %r15,%r15 0xffffffff82298e4d <dbuf_destroy+109>: je 0xffffffff82298e8a <dbuf_destroy+170> 0xffffffff82298e4f <dbuf_destroy+111>: movq $0x0,0xd8(%r13) 0xffffffff82298e5a <dbuf_destroy+122>: mov 0x30(%r15),%rax 0xffffffff82298e5e <dbuf_destroy+126>: mov 0x38(%r15),%rbx 0xffffffff82298e62 <dbuf_destroy+130>: test %rax,%rax 0xffffffff82298e65 <dbuf_destroy+133>: je 0xffffffff82298e6c <dbuf_destroy+140> 0xffffffff82298e67 <dbuf_destroy+135>: mov %r15,%rdi 0xffffffff82298e6a <dbuf_destroy+138>: callq *%rax 0xffffffff82298e6c <dbuf_destroy+140>: test %rbx,%rbx 0xffffffff82298e6f <dbuf_destroy+143>: je 0xffffffff82298e8a <dbuf_destroy+170> 0xffffffff82298e71 <dbuf_destroy+145>: mov 0xffffffff8240c470,%rdi 0xffffffff82298e79 <dbuf_destroy+153>: mov 0x38(%r15),%rsi 0xffffffff82298e7d <dbuf_destroy+157>: xor %ecx,%ecx 0xffffffff82298e7f <dbuf_destroy+159>: mov %r15,%rdx 0xffffffff82298e82 <dbuf_destroy+162>: mov %r15,%r8 0xffffffff82298e85 <dbuf_destroy+165>: callq 0xffffffff82272960 <taskq_dispatch_ent> 0xffffffff82298e8a <dbuf_destroy+170>: movq $0x0,0x18(%r13) 0xffffffff82298e92 <dbuf_destroy+178>: cmpl $0x2,0x78(%r13) 0xffffffff82298e97 <dbuf_destroy+183>: je 0xffffffff82298ea1 <dbuf_destroy+193> 0xffffffff82298e99 <dbuf_destroy+185>: movl $0x0,0x78(%r13) 0xffffffff82298ea1 <dbuf_destroy+193>: lea 0xc8(%r13),%rdi 0xffffffff82298ea8 <dbuf_destroy+200>: callq 0xffffffff822dbf30 <multilist_link_active> 0xffffffff82298ead <dbuf_destroy+205>: test %eax,%eax 0xffffffff82298eaf <dbuf_destroy+207>: je 0xffffffff82298ed4 <dbuf_destroy+244> 0xffffffff82298eb1 <dbuf_destroy+209>: mov 0xffffffff8240c478,%rdi 0xffffffff82298eb9 <dbuf_destroy+217>: mov %r13,%rsi 0xffffffff82298ebc <dbuf_destroy+220>: callq 0xffffffff822dbbe0 <multilist_remove> 0xffffffff82298ec1 <dbuf_destroy+225>: mov 0x10(%r13),%rsi 0xffffffff82298ec5 <dbuf_destroy+229>: neg %rsi 0xffffffff82298ec8 <dbuf_destroy+232>: mov $0xffffffff8240c480,%rdi 0xffffffff82298ecf <dbuf_destroy+239>: callq 0xffffffff82273960 <atomic_add_64_nv> 0xffffffff82298ed4 <dbuf_destroy+244>: movl $0x5,0x78(%r13) 0xffffffff82298edc <dbuf_destroy+252>: movq $0x0,0x48(%r13) 0xffffffff82298ee4 <dbuf_destroy+260>: lea 0x58(%r13),%rdi 0xffffffff82298ee8 <dbuf_destroy+264>: mov $0xffffffff823d4fd1,%rsi 0xffffffff82298eef <dbuf_destroy+271>: mov $0x812,%edx 0xffffffff82298ef4 <dbuf_destroy+276>: callq 0xffffffff80aff910 <_sx_xunl= ock> 0xffffffff82298ef9 <dbuf_destroy+281>: mov 0x28(%r13),%rdi 0xffffffff82298efd <dbuf_destroy+285>: mov $0xffffffff823d5126,%rsi 0xffffffff82298f04 <dbuf_destroy+292>: callq 0xffffffff82331f70 <zrl_add_impl> 0xffffffff82298f09 <dbuf_destroy+297>: mov 0x28(%r13),%rdi 0xffffffff82298f0d <dbuf_destroy+301>: mov 0x40(%rdi),%r15 0xffffffff82298f11 <dbuf_destroy+305>: mov 0x40(%r15),%rbx 0xffffffff82298f15 <dbuf_destroy+309>: cmpq $0xffffffffffffffff,0x40(%r1= 3) 0xffffffff82298f1a <dbuf_destroy+314>: je 0xffffffff82299059 <dbuf_destroy+633> 0xffffffff82298f20 <dbuf_destroy+320>: mov %rbx,-0x30(%rbp) 0xffffffff82298f24 <dbuf_destroy+324>: mov %r14,-0x38(%rbp) 0xffffffff82298f28 <dbuf_destroy+328>: lea 0x1f8(%r15),%r12 0xffffffff82298f2f <dbuf_destroy+335>: mov 0x210(%r15),%rbx 0xffffffff82298f36 <dbuf_destroy+342>: and $0xfffffffffffffff1,%rbx 0xffffffff82298f3a <dbuf_destroy+346>: mov %gs:0x0,%r14 0xffffffff82298f43 <dbuf_destroy+355>: cmp %r14,%rbx 0xffffffff82298f46 <dbuf_destroy+358>: je 0xffffffff82298f5e <dbuf_destroy+382> 0xffffffff82298f48 <dbuf_destroy+360>: xor %esi,%esi 0xffffffff82298f4a <dbuf_destroy+362>: mov $0xffffffff823d4fd1,%rdx 0xffffffff82298f51 <dbuf_destroy+369>: mov $0x81a,%ecx 0xffffffff82298f56 <dbuf_destroy+374>: mov %r12,%rdi 0xffffffff82298f59 <dbuf_destroy+377>: callq 0xffffffff80aff0d0 <_sx_xloc= k> 0xffffffff82298f5e <dbuf_destroy+382>: lea 0x218(%r15),%rdi 0xffffffff82298f65 <dbuf_destroy+389>: mov %r13,%rsi 0xffffffff82298f68 <dbuf_destroy+392>: callq 0xffffffff82266e70 <avl_remo= ve> 0xffffffff82298f6d <dbuf_destroy+397>: lea 0xa8(%r15),%rdi 0xffffffff82298f74 <dbuf_destroy+404>: mov $0x1,%esi 0xffffffff82298f79 <dbuf_destroy+409>: callq 0xffffffff80f56de0 <atomic_subtract_int> 0xffffffff82298f7e <dbuf_destroy+414>: callq 0xffffffff822739b0 <membar_producer> 0xffffffff82298f83 <dbuf_destroy+419>: mov 0x28(%r13),%rdi 0xffffffff82298f87 <dbuf_destroy+423>: callq 0xffffffff82332000 <zrl_remo= ve> 0xffffffff82298f8c <dbuf_destroy+428>: cmp %r14,%rbx 0xffffffff82298f8f <dbuf_destroy+431>: je 0xffffffff82298fa5 <dbuf_destroy+453> 0xffffffff82298f91 <dbuf_destroy+433>: mov $0xffffffff823d4fd1,%rsi 0xffffffff82298f98 <dbuf_destroy+440>: mov $0x820,%edx 0xffffffff82298f9d <dbuf_destroy+445>: mov %r12,%rdi 0xffffffff82298fa0 <dbuf_destroy+448>: callq 0xffffffff80aff910 <_sx_xunl= ock> 0xffffffff82298fa5 <dbuf_destroy+453>: mov %r15,%rdi 0xffffffff82298fa8 <dbuf_destroy+456>: mov %r13,%rsi 0xffffffff82298fab <dbuf_destroy+459>: callq 0xffffffff822b4dd0 <dnode_re= le> 0xffffffff82298fb0 <dbuf_destroy+464>: movq $0x0,0x28(%r13) 0xffffffff82298fb8 <dbuf_destroy+472>: mov 0x0(%r13),%rsi 0xffffffff82298fbc <dbuf_destroy+476>: mov 0x20(%r13),%rdi 0xffffffff82298fc0 <dbuf_destroy+480>: mov 0x40(%r13),%rcx 0xffffffff82298fc4 <dbuf_destroy+484>: movzbl 0x50(%r13),%edx 0xffffffff82298fc9 <dbuf_destroy+489>: callq 0xffffffff82297340 <cityhash= 4> 0xffffffff82298fce <dbuf_destroy+494>: mov %rax,%rbx 0xffffffff82298fd1 <dbuf_destroy+497>: and 0xffffffff8240a458,%rbx 0xffffffff82298fd9 <dbuf_destroy+505>: movzbl %bl,%eax 0xffffffff82298fdc <dbuf_destroy+508>: shl $0x5,%rax 0xffffffff82298fe0 <dbuf_destroy+512>: lea -0x7dbf5b98(%rax),%r15 0xffffffff82298fe7 <dbuf_destroy+519>: xor %esi,%esi 0xffffffff82298fe9 <dbuf_destroy+521>: mov $0xffffffff823d4fd1,%rdx 0xffffffff82298ff0 <dbuf_destroy+528>: mov $0x129,%ecx 0xffffffff82298ff5 <dbuf_destroy+533>: mov %r15,%rdi 0xffffffff82298ff8 <dbuf_destroy+536>: callq 0xffffffff80aff0d0 <_sx_xloc= k> 0xffffffff82298ffd <dbuf_destroy+541>: shl $0x3,%rbx 0xffffffff82299001 <dbuf_destroy+545>: add 0xffffffff8240a460,%rbx 0xffffffff82299009 <dbuf_destroy+553>: mov -0x38(%rbp),%r14 0xffffffff8229900d <dbuf_destroy+557>: nopl (%rax) 0xffffffff82299010 <dbuf_destroy+560>: mov %rbx,%rax 0xffffffff82299013 <dbuf_destroy+563>: mov (%rax),%rcx 0xffffffff82299016 <dbuf_destroy+566>: lea 0x38(%rcx),%rbx 0xffffffff8229901a <dbuf_destroy+570>: cmp %r13,%rcx 0xffffffff8229901d <dbuf_destroy+573>: jne 0xffffffff82299010 <dbuf_destroy+560> 0xffffffff8229901f <dbuf_destroy+575>: mov 0x38(%r13),%rcx 0xffffffff82299023 <dbuf_destroy+579>: mov %rcx,(%rax) 0xffffffff82299026 <dbuf_destroy+582>: movq $0x0,0x38(%r13) 0xffffffff8229902e <dbuf_destroy+590>: mov $0xffffffff823d4fd1,%rsi 0xffffffff82299035 <dbuf_destroy+597>: mov $0x131,%edx 0xffffffff8229903a <dbuf_destroy+602>: mov %r15,%rdi 0xffffffff8229903d <dbuf_destroy+605>: callq 0xffffffff80aff910 <_sx_xunl= ock> 0xffffffff82299042 <dbuf_destroy+610>: mov $0xffffffff8240c4c8,%rdi 0xffffffff82299049 <dbuf_destroy+617>: mov $0x1,%esi 0xffffffff8229904e <dbuf_destroy+622>: callq 0xffffffff80f56e60 <atomic_subtract_long> 0xffffffff82299053 <dbuf_destroy+627>: mov -0x30(%rbp),%rbx 0xffffffff82299057 <dbuf_destroy+631>: jmp 0xffffffff8229905e <dbuf_destroy+638> 0xffffffff82299059 <dbuf_destroy+633>: callq 0xffffffff82332000 <zrl_remo= ve> 0xffffffff8229905e <dbuf_destroy+638>: movq $0x0,0x30(%r13) 0xffffffff82299066 <dbuf_destroy+646>: mov 0xffffffff8240c468,%rdi 0xffffffff8229906e <dbuf_destroy+654>: mov %r13,%rsi 0xffffffff82299071 <dbuf_destroy+657>: callq 0xffffffff825e83c0 <kmem_cache_free> 0xffffffff82299076 <dbuf_destroy+662>: mov $0xe8,%edi 0xffffffff8229907b <dbuf_destroy+667>: mov $0x4,%esi 0xffffffff82299080 <dbuf_destroy+672>: callq 0xffffffff8228b6c0 <arc_space_return> 0xffffffff82299085 <dbuf_destroy+677>: test %r14,%r14 0xffffffff82299088 <dbuf_destroy+680>: je 0xffffffff822990bc <dbuf_destroy+732> 0xffffffff8229908a <dbuf_destroy+682>: cmp %rbx,%r14 0xffffffff8229908d <dbuf_destroy+685>: je 0xffffffff822990bc <dbuf_destroy+732> 0xffffffff8229908f <dbuf_destroy+687>: lea 0x58(%r14),%rdi 0xffffffff82299093 <dbuf_destroy+691>: xor %esi,%esi 0xffffffff82299095 <dbuf_destroy+693>: mov $0xffffffff823d4fd1,%rdx 0xffffffff8229909c <dbuf_destroy+700>: mov $0xaa6,%ecx 0xffffffff822990a1 <dbuf_destroy+705>: callq 0xffffffff80aff0d0 <_sx_xloc= k> 0xffffffff822990a6 <dbuf_destroy+710>: mov %r14,%rdi 0xffffffff822990a9 <dbuf_destroy+713>: add $0x18,%rsp 0xffffffff822990ad <dbuf_destroy+717>: pop %rbx 0xffffffff822990ae <dbuf_destroy+718>: pop %r12 0xffffffff822990b0 <dbuf_destroy+720>: pop %r13 0xffffffff822990b2 <dbuf_destroy+722>: pop %r14 0xffffffff822990b4 <dbuf_destroy+724>: pop %r15 0xffffffff822990b6 <dbuf_destroy+726>: pop %rbp 0xffffffff822990b7 <dbuf_destroy+727>: jmpq 0xffffffff8229b290 <dbuf_rele_and_unlock> 0xffffffff822990bc <dbuf_destroy+732>: add $0x18,%rsp 0xffffffff822990c0 <dbuf_destroy+736>: pop %rbx 0xffffffff822990c1 <dbuf_destroy+737>: pop %r12 0xffffffff822990c3 <dbuf_destroy+739>: pop %r13 0xffffffff822990c5 <dbuf_destroy+741>: pop %r14 0xffffffff822990c7 <dbuf_destroy+743>: pop %r15 0xffffffff822990c9 <dbuf_destroy+745>: pop %rbp 0xffffffff822990ca <dbuf_destroy+746>: retq End of assembler dump. Current language: auto; currently minimal --=20 You are receiving this mail because: You are the assignee for the bug.=
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?bug-235783-3630-nuDCDTR8Wu>