Skip site navigation (1)Skip section navigation (2) 
Date:      Mon, 02 Jun 2014 13:46:43 -0700
From:      Mike Carlson <mike@bayphoto.com>
To:        Steven Hartland <killing@multiplay.co.uk>, freebsd-fs@freebsd.org
Subject:   Re: ZFS Kernel Panic on 10.0-RELEASE
Message-ID:  <538CE2B3.8090008@bayphoto.com>
In-Reply-To: <88B3A7562A5F4F9B9EEF0E83BCAD2FB0@multiplay.co.uk>
References:  <5388D64D.4030400@bayphoto.com> <EC2EA442-56FC-46B4-A1E2-97523029B7B3@mail.turbofuzz.com> <5388E5B4.3030002@bayphoto.com> <538BBEB7.4070008@bayphoto.com> <782C34792E95484DBA631A96FE3BEF20@multiplay.co.uk> <538C9CF3.6070208@bayphoto.com> <16ADD4D9DC73403C9669D8F34FDBD316@multiplay.co.uk> <538CB3EA.9010807@bayphoto.com> <6C6FB182781541CEBF627998B73B1DB4@multiplay.co.uk> <538CC16A.6060207@bayphoto.com> <F959477921CD4552A94BF932A55961F4@multiplay.co.uk> <538CDB7F.2060408@bayphoto.com> <88B3A7562A5F4F9B9EEF0E83BCAD2FB0@multiplay.co.uk>
next in thread | previous in thread | raw e-mail | index | archive | help 

[-- Attachment #1 --]
On 6/2/2014 1:44 PM, Steven Hartland wrote:
>
> ----- Original Message ----- From: "Mike Carlson" <mike@bayphoto.com>
> To: "Steven Hartland" <killing@multiplay.co.uk>; <freebsd-fs@freebsd.org>
> Sent: Monday, June 02, 2014 9:15 PM
> Subject: Re: ZFS Kernel Panic on 10.0-RELEASE
>
>
>> On 6/2/2014 1:06 PM, Steven Hartland wrote:
>
>>> I don't have a core.0.txt, I only have:
>>>
>>>      ~/p/z/dump> ls -al
>>>    total 347690
>>>    drwxr-xr-x  3 mikec  wheel              8 Jun  2 03:25 .
>>>    drwxr-xr-x  4 mikec  wheel              5 Jun  2 10:44 ..
>>>    drwxrwxr-x  2 mikec  operator           2 Jun  2 03:07 .snap
>>>    -rw-r--r--  1 mikec  wheel              2 Jun  2 03:24 bounds
>>>    -rw-------  1 mikec  wheel            446 Jun  2 03:24 info.0
>>>    lrwxr-xr-x  1 mikec  wheel              6 Jun  2 03:25 info.last ->
>>>    info.0
>>>    -rw-------  1 mikec  wheel     3469885440 Jun  2 03:25 vmcore.0
>>>    lrwxr-xr-x  1 mikec  wheel              8 Jun  2 03:25 vmcore.last
>>>    -> vmcore.0
>>>
>>> But, here is the kgdb output (with backtrace):
>>>
>>>      ~/p/z/dump> cat ../kgdb_backtrace.txt
>>>    <118>root@:/ # zfs set canmount=on zroot/data/working
>>>    <118>root@:/ # zfs mount zroot/data/working
>>>
>>>
>>>    Fatal trap 12: page fault while in kernel mode
>>>    cpuid = 14; apic id = 22
>>>    fault virtual address   = 0x4a0
>>>    fault code              = supervisor read data, page not present
>>>    instruction pointer     = 0x20:0xffffffff8185a39f
>>>    stack pointer           = 0x28:0xfffffe1834608570
>>>    frame pointer           = 0x28:0xfffffe18346085b0
>>>    code segment            = base 0x0, limit 0xfffff, type 0x1b
>>>                             = DPL 0, pres 1, long 1, def32 0, gran 1
>>>    processor eflags        = interrupt enabled, resume, IOPL = 0
>>>    current process         = 2 (txg_thread_enter)
>>>    trap number             = 12
>>>    panic: page fault
>>>    cpuid = 14
>>>    KDB: stack backtrace:
>>>    #0 0xffffffff808e7ee0 at kdb_backtrace+0x60
>>>    #1 0xffffffff808af9c5 at panic+0x155
>>>    #2 0xffffffff80c8e7b2 at trap_fatal+0x3a2
>>>    #3 0xffffffff80c8ea89 at trap_pfault+0x2c9
>>>    #4 0xffffffff80c8e216 at trap+0x5e6
>>>    #5 0xffffffff80c754b2 at calltrap+0x8
>>>    #6 0xffffffff8182eb5a at dsl_dataset_block_kill+0x3a
>>>    #7 0xffffffff8182b967 at dnode_sync+0x237
>>>    #8 0xffffffff81823fcb at dmu_objset_sync_dnodes+0x2b
>>>    #9 0xffffffff81823e4d at dmu_objset_sync+0x1ed
>>>    #10 0xffffffff8183829a at dsl_pool_sync+0xca
>>>    #11 0xffffffff81853a4e at spa_sync+0x52e
>>>    #12 0xffffffff8185c925 at txg_sync_thread+0x375
>>>    #13 0xffffffff80881a9a at fork_exit+0x9a
>>>    #14 0xffffffff80c759ee at fork_trampoline+0xe
>>>    Uptime: 26m15s
>>>    Dumping 3309 out of 98234
>>>    MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%
>>>
>>>    Reading symbols from /boot/kernel/zfs.ko.symbols...done.
>>>    Loaded symbols for /boot/kernel/zfs.ko.symbols
>>>    Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
>>>    Loaded symbols for /boot/kernel/opensolaris.ko.symbols
>>>    #0  doadump (textdump=<value optimized out>) at pcpu.h:219
>>>    219             __asm("movq %%gs:%1,%0" : "=r" (td)
>>>    (kgdb) backtrace
>>>    #0  doadump (textdump=<value optimized out>) at pcpu.h:219
>>>    #1  0xffffffff808af640 in kern_reboot (howto=260) at
>>>    /usr/src/sys/kern/kern_shutdown.c:447
>>>    #2  0xffffffff808afa04 in panic (fmt=<value optimized out>) at
>>>    /usr/src/sys/kern/kern_shutdown.c:754
>>>    #3  0xffffffff80c8e7b2 in trap_fatal (frame=<value optimized out>,
>>>    eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:882
>>>    #4  0xffffffff80c8ea89 in trap_pfault (frame=0xfffffe18346084c0,
>>>    usermode=0) at /usr/src/sys/amd64/amd64/trap.c:699
>>>    #5  0xffffffff80c8e216 in trap (frame=0xfffffe18346084c0) at
>>>    /usr/src/sys/amd64/amd64/trap.c:463
>>>    #6  0xffffffff80c754b2 in calltrap () at
>>>    /usr/src/sys/amd64/amd64/exception.S:232
>>>    #7  0xffffffff8185a39f in bp_get_dsize_sync (spa=0xfffff80041835000,
>>>    bp=0xfffffe001b8a1780)
>>>         at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:1635 
>>>
>>>    #8  0xffffffff8182eb5a in dsl_dataset_block_kill
>>>    (ds=0xfffff800410fec00, bp=0xfffffe001b8a1780,
>>>    tx=0xfffff8004faa0600, async=0)
>>>         at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c:129 
>>>
>>>    #9  0xffffffff8182b967 in dnode_sync (dn=0xfffff8004fe626c0,
>>>    tx=0xfffff8004faa0600) at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c:128 
>>>
>>>    #10 0xffffffff81823fcb in dmu_objset_sync_dnodes
>>>    (list=0xfffff80041956b10, newlist=<value optimized out>, tx=<value
>>>    optimized out>)
>>>         at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c:945 
>>>
>>>    #11 0xffffffff81823e4d in dmu_objset_sync (os=0xfffff80041956800,
>>>    pio=0xfffff800418c43b0, tx=0xfffff8004faa0600)
>>>         at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c:1062 
>>>
>>>    #12 0xffffffff8183829a in dsl_pool_sync (dp=0xfffff8004183c000,
>>>    txg=<value optimized out>)
>>>         at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c:413 
>>>
>>>    #13 0xffffffff81853a4e in spa_sync (spa=0xfffff80041835000,
>>>    txg=3373534) at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6410 
>>>
>>>    #14 0xffffffff8185c925 in txg_sync_thread (arg=0xfffff8004183c000)
>>>    at
>>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:515 
>>>
>>>    #15 0xffffffff80881a9a in fork_exit (callout=0xffffffff8185c5b0
>>>    <txg_sync_thread>, arg=0xfffff8004183c000, frame=0xfffffe1834608ac0)
>>>    at /usr/src/sys/kern/kern_fork.c:995
>>>    #16 0xffffffff80c759ee in fork_trampoline () at
>>>    /usr/src/sys/amd64/amd64/exception.S:606
>>>    #17 0x0000000000000000 in ?? ()
>>>    Current language:  auto; currently minimal
>>>
>>>
>>> If anyone wants to help out and check out the vmcore file, email me 
>>> off the list and I'll provide a S3 url of the tar'd + xz file.
>>>
>>>
>> Output of "frame 7":
>>
>>    (kgdb) frame 7
>>    #7  0xffffffff8185a39f in bp_get_dsize_sync (spa=0xfffff80041835000,
>>    bp=0xfffffe001b8a1780)
>>         at
>> /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:1635
>>    1635                    dsize = (asize >> SPA_MINBLOCKSHIFT) *
>>    vd->vdev_deflate_ratio;
>>
>> Is that what you were looking for?
>
> Thats the line I gathered it was on but no I need to know what the value
> of vd is, so what you need to do is:
> print vd
>
> If thats valid then:
> print *vd
>
It reports:

(kgdb) print *vd
No symbol "vd" in current context.

Should I rebuild the kernel with additional options?
> Given the panic I'm kind of expecting garbage or null (0x00)
>
>> I'm not familar with this process, so I hope this does not become too 
>> painful in pulling the details out.
>
> No problem, everyone has to learn some time ;-)
>

Thanks Steve :)
> Regards
> Steve
>
>



[-- Attachment #2 --]
0	*H
010	+0	*H
"00e3v=0
	*H
0K10
URootCA10U

Bay Photo Lab10U
California10	UUS0
121023173218Z
271023173218Z0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUS0"0
	*H
0
;TąuyK~Zz2M'4
EiTj)yL5"kv7Urn \!SgP;zh>ˊj\VovX<LgfxxkL1CdY\S;z(5TO[)5bu\mBj*
nUh&`Qί;ZxȜF
ԧ@})8}4#dzw&P^=AdT}*4 qS^E)̈cA$XDS]Z/_5M`~ӻRo'Ftw\e.G.3@m"\,c{'Gidv(TQY9zbpˆ9c#Y³Vs|If	ew7I%Grau07hf.;{Jʾx/R1.LT}Կ!kb
o8H	]=}SΈ퉃00U-rfbb,v0U00U#0FNqi$x'{(W+0U00}{http://bayca.bayhoto.local/ejbca/publicweb/webdist/certdist?cmd=crl&issuer=CN=RootCA,O=Bay%20Photo%20Lab,ST=California,C=US0U0
	*H
JMUZ>7gm[z }/.~^J;өƉ-Q_\Όh޲#ԾXL7ph(@`+8W&ib!Qj+ȡ1iT(#^( giZ9c<R꼓e.ݘVѬ峿ۅ8Dh$~mm啠~'\ET&	 a}rMKL0u%HYL
l=`Υ3k[؝Y}$ ss8?~IXKda<==mL[RҠsHBR/*`JfUzA)'0JkArvp#e-{]U
Z`#2Ϡv~.#l7"D=&t^-Q_9Mi
uԒn{Zn!U%r3J;Q׼Di@PNg]&;yw|9B*.L=Ij-)/]'g^U0#0b=0
	*H
0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUS0
121023180003Z
141023180003Z0`10
	&,d1306910UMike Carlson10	UIT10U

Bay Photo Lab10	UUS0"0
	*H
0
<ȼ^|=e9KtФ-jI_	%[߲'O%3;=*n((RT	•͐C/\WU@HCjrIU-iE˼|paҨm-4݈amƵbK$"UEkEzd
w.
wG u:B'9!?tdk%%̞N.8C1ަί[
BjF0){C9&pXnĉZuX")3zsS\\D:L׏1Q}1Gzz(d#V3fRoш^CLfQ@S/StX
d5Y3M0ՙQ5ō;pIdV]&d#26zsgM}r#iМ|3)md:}뚁00R+F0D0B+06http://bayca.bayhoto.local/ejbca/publicweb/status/ocsp0U(}awJ״(#0U00U#0-rfbb,v0U00U%0
+0U0mike@bayphoto.com0
	*H
9|&V,*Hd	ƏA~6fFg'^y
I'yy,v}Z	@ᔘ7\F5QA37*LT4VStTe .Dӧ=n}=L\E	{
z7kYs#RO}E`OnL'1M0`Dۋ
rvVuX?s=	+O0:yE?BA̡5|Ʀpp*<FLA36k덝j9b=&)KJSmʐXo@g;V4@ujkX9	@Wh#nl\Y)A
rFGjqtvhu.ճK)L}@41AKz&ȴztÈ6͢j=0*+@;xnc-
WƣLG9X)=
y%]Q@BW
,Άut00MiC0
	*H
0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUS0
121023175745Z
141023175745Z0`10
	&,d1306910UMike Carlson10	UIT10U

Bay Photo Lab10	UUS0"0
	*H
0
@vɌAļVAW5:eh$n>b%k7Pwޡ=^CBv2ULLqn6+>A:P#=ѕ[8Z<|&wb(x椉
iҒx9H?~Ɔ-y]jN崡1geAˇwH4w?h!/^Pؕfa5-+%<*/+`ZBCƀno|6'zoe!)@H藱$zѩ+
SXDz(~Bݬe?V
\j;.P,銉[JݦkjY*nȡ5]
hlkz3.Wme/tɧ#	8L%
Ũ%zp	_p)ڜ(C=MYe3S>Tfρ=@	]ڑav&0ۗ;.j'Yk_00R+F0D0B+06http://bayca.bayhoto.local/ejbca/publicweb/status/ocsp0UFO+Rdb`?60U00U#0-rfbb,v0U00http://bayca.bayhoto.local/ejbca/publicweb/webdist/certdist?cmd=crl&issuer=CN=Bay%20Photo%20People%20CA,O=Bay%20Photo%20Lab,ST=California,C=US0U0U%0++0U0mike@bayphoto.com0
	*H
/ungfsy@KLw.cM&6?-Y4++IJYD	C£S_2$eڏPU((̖S~aM0ri~jk2Ւ[n9rn&Bz(MݼIܪ*ȱImu5lr[Q`3͈;l{Z07h$>at)qo\]pJW7*[c%

y1FB)p2͞[~=?!Wd9XY5.bOKUDV[Z98E
^X9n<Hi@C?H+jlۗc݌&yqQ<Ii/
ɣ*B!f<.Re-=Y*?-4;|vj1@+Iܑ=J7%'jMmrSM@GV|:C'ݮ_Lkt61F0B0d0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUSMiC0	+0	*H
	1	*H
0	*H
	1
140602204643Z0#	*H
	14EDP+r<(!踥h0l	*H
	1_0]0	`He*0	`He0
*H
0*H
0
*H
@0+0
*H
(0s	+71f0d0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUSb=0u*H
	1fd0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUSb=0
	*H
˴v2'$,0>
QN~A	ĩ""#NgN
oob-?/r*(Gn,6An-V81RLxޞUQV+

5~^'1$fgg$nhYj6&J,uf~=G_Q;m/<$%]QrUTY)ٷow-v{o3=Y8t9fwm^(ɮ~h'uIڬ<-;ÞY{ȝ7kg!
-UěoܘO}Kֱ-CI`тdq5F].p];2R-hχnMr?y`RqTH,eA݋xeC.zPC,oSv.38!ZSOHUˇ{cP

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?538CE2B3.8090008>