Skip site navigation (1)Skip section navigation (2) 
Date:      Mon, 02 Jun 2014 11:24:42 -0700
From:      Mike Carlson <mike@bayphoto.com>
To:        Steven Hartland <killing@multiplay.co.uk>, freebsd-fs@freebsd.org
Subject:   Re: ZFS Kernel Panic on 10.0-RELEASE
Message-ID:  <538CC16A.6060207@bayphoto.com>
In-Reply-To: <6C6FB182781541CEBF627998B73B1DB4@multiplay.co.uk>
References:  <5388D64D.4030400@bayphoto.com> <EC2EA442-56FC-46B4-A1E2-97523029B7B3@mail.turbofuzz.com> <5388E5B4.3030002@bayphoto.com> <538BBEB7.4070008@bayphoto.com> <782C34792E95484DBA631A96FE3BEF20@multiplay.co.uk> <538C9CF3.6070208@bayphoto.com> <16ADD4D9DC73403C9669D8F34FDBD316@multiplay.co.uk> <538CB3EA.9010807@bayphoto.com> <6C6FB182781541CEBF627998B73B1DB4@multiplay.co.uk>
next in thread | previous in thread | raw e-mail | index | archive | help 

[-- Attachment #1 --]
On 6/2/2014 10:35 AM, Steven Hartland wrote:
> ----- Original Message ----- From: "Mike Carlson" <mike@bayphoto.com>
>> I have a crash dump, its 3.2GB in size.
>>
>> How can I get this to the right people? I can put it up on S3, but 
>> I'd prefer to limit its access since it may contain sensitive 
>> information (we dont have local accounts aside from root, but better 
>> to be safe)
>
> For starters can you put the top contents of core.X.txt somewhere.
>
> If there's sensitive info in there just the top kgdb section with
> the backtrace will be good initially.
>
>    Regards
>    Steve
>
>
I don't have a core.0.txt, I only have:

      ~/p/z/dump> ls -al
    total 347690
    drwxr-xr-x  3 mikec  wheel              8 Jun  2 03:25 .
    drwxr-xr-x  4 mikec  wheel              5 Jun  2 10:44 ..
    drwxrwxr-x  2 mikec  operator           2 Jun  2 03:07 .snap
    -rw-r--r--  1 mikec  wheel              2 Jun  2 03:24 bounds
    -rw-------  1 mikec  wheel            446 Jun  2 03:24 info.0
    lrwxr-xr-x  1 mikec  wheel              6 Jun  2 03:25 info.last ->
    info.0
    -rw-------  1 mikec  wheel     3469885440 Jun  2 03:25 vmcore.0
    lrwxr-xr-x  1 mikec  wheel              8 Jun  2 03:25 vmcore.last
    -> vmcore.0

But, here is the kgdb output (with backtrace):

      ~/p/z/dump> cat ../kgdb_backtrace.txt
    <118>root@:/ # zfs set canmount=on zroot/data/working
    <118>root@:/ # zfs mount zroot/data/working


    Fatal trap 12: page fault while in kernel mode
    cpuid = 14; apic id = 22
    fault virtual address   = 0x4a0
    fault code              = supervisor read data, page not present
    instruction pointer     = 0x20:0xffffffff8185a39f
    stack pointer           = 0x28:0xfffffe1834608570
    frame pointer           = 0x28:0xfffffe18346085b0
    code segment            = base 0x0, limit 0xfffff, type 0x1b
                             = DPL 0, pres 1, long 1, def32 0, gran 1
    processor eflags        = interrupt enabled, resume, IOPL = 0
    current process         = 2 (txg_thread_enter)
    trap number             = 12
    panic: page fault
    cpuid = 14
    KDB: stack backtrace:
    #0 0xffffffff808e7ee0 at kdb_backtrace+0x60
    #1 0xffffffff808af9c5 at panic+0x155
    #2 0xffffffff80c8e7b2 at trap_fatal+0x3a2
    #3 0xffffffff80c8ea89 at trap_pfault+0x2c9
    #4 0xffffffff80c8e216 at trap+0x5e6
    #5 0xffffffff80c754b2 at calltrap+0x8
    #6 0xffffffff8182eb5a at dsl_dataset_block_kill+0x3a
    #7 0xffffffff8182b967 at dnode_sync+0x237
    #8 0xffffffff81823fcb at dmu_objset_sync_dnodes+0x2b
    #9 0xffffffff81823e4d at dmu_objset_sync+0x1ed
    #10 0xffffffff8183829a at dsl_pool_sync+0xca
    #11 0xffffffff81853a4e at spa_sync+0x52e
    #12 0xffffffff8185c925 at txg_sync_thread+0x375
    #13 0xffffffff80881a9a at fork_exit+0x9a
    #14 0xffffffff80c759ee at fork_trampoline+0xe
    Uptime: 26m15s
    Dumping 3309 out of 98234
    MB:..1%..11%..21%..31%..41%..51%..61%..71%..81%..91%

    Reading symbols from /boot/kernel/zfs.ko.symbols...done.
    Loaded symbols for /boot/kernel/zfs.ko.symbols
    Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
    Loaded symbols for /boot/kernel/opensolaris.ko.symbols
    #0  doadump (textdump=<value optimized out>) at pcpu.h:219
    219             __asm("movq %%gs:%1,%0" : "=r" (td)
    (kgdb) backtrace
    #0  doadump (textdump=<value optimized out>) at pcpu.h:219
    #1  0xffffffff808af640 in kern_reboot (howto=260) at
    /usr/src/sys/kern/kern_shutdown.c:447
    #2  0xffffffff808afa04 in panic (fmt=<value optimized out>) at
    /usr/src/sys/kern/kern_shutdown.c:754
    #3  0xffffffff80c8e7b2 in trap_fatal (frame=<value optimized out>,
    eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:882
    #4  0xffffffff80c8ea89 in trap_pfault (frame=0xfffffe18346084c0,
    usermode=0) at /usr/src/sys/amd64/amd64/trap.c:699
    #5  0xffffffff80c8e216 in trap (frame=0xfffffe18346084c0) at
    /usr/src/sys/amd64/amd64/trap.c:463
    #6  0xffffffff80c754b2 in calltrap () at
    /usr/src/sys/amd64/amd64/exception.S:232
    #7  0xffffffff8185a39f in bp_get_dsize_sync (spa=0xfffff80041835000,
    bp=0xfffffe001b8a1780)
         at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/spa_misc.c:1635
    #8  0xffffffff8182eb5a in dsl_dataset_block_kill
    (ds=0xfffff800410fec00, bp=0xfffffe001b8a1780,
    tx=0xfffff8004faa0600, async=0)
         at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c:129
    #9  0xffffffff8182b967 in dnode_sync (dn=0xfffff8004fe626c0,
    tx=0xfffff8004faa0600) at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dnode_sync.c:128
    #10 0xffffffff81823fcb in dmu_objset_sync_dnodes
    (list=0xfffff80041956b10, newlist=<value optimized out>, tx=<value
    optimized out>)
         at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c:945
    #11 0xffffffff81823e4d in dmu_objset_sync (os=0xfffff80041956800,
    pio=0xfffff800418c43b0, tx=0xfffff8004faa0600)
         at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dmu_objset.c:1062
    #12 0xffffffff8183829a in dsl_pool_sync (dp=0xfffff8004183c000,
    txg=<value optimized out>)
         at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_pool.c:413
    #13 0xffffffff81853a4e in spa_sync (spa=0xfffff80041835000,
    txg=3373534) at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c:6410
    #14 0xffffffff8185c925 in txg_sync_thread (arg=0xfffff8004183c000)
    at
    /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/fs/zfs/txg.c:515
    #15 0xffffffff80881a9a in fork_exit (callout=0xffffffff8185c5b0
    <txg_sync_thread>, arg=0xfffff8004183c000, frame=0xfffffe1834608ac0)
    at /usr/src/sys/kern/kern_fork.c:995
    #16 0xffffffff80c759ee in fork_trampoline () at
    /usr/src/sys/amd64/amd64/exception.S:606
    #17 0x0000000000000000 in ?? ()
    Current language:  auto; currently minimal


If anyone wants to help out and check out the vmcore file, email me off 
the list and I'll provide a S3 url of the tar'd + xz file.

Thanks,
Mike C

[-- Attachment #2 --]
0	*H
010	+0	*H
"00e3v=0
	*H
0K10
URootCA10U

Bay Photo Lab10U
California10	UUS0
121023173218Z
271023173218Z0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUS0"0
	*H
0
;TąuyK~Zz2M'4
EiTj)yL5"kv7Urn \!SgP;zh>ˊj\VovX<LgfxxkL1CdY\S;z(5TO[)5bu\mBj*
nUh&`Qί;ZxȜF
ԧ@})8}4#dzw&P^=AdT}*4 qS^E)̈cA$XDS]Z/_5M`~ӻRo'Ftw\e.G.3@m"\,c{'Gidv(TQY9zbpˆ9c#Y³Vs|If	ew7I%Grau07hf.;{Jʾx/R1.LT}Կ!kb
o8H	]=}SΈ퉃00U-rfbb,v0U00U#0FNqi$x'{(W+0U00}{http://bayca.bayhoto.local/ejbca/publicweb/webdist/certdist?cmd=crl&issuer=CN=RootCA,O=Bay%20Photo%20Lab,ST=California,C=US0U0
	*H
JMUZ>7gm[z }/.~^J;өƉ-Q_\Όh޲#ԾXL7ph(@`+8W&ib!Qj+ȡ1iT(#^( giZ9c<R꼓e.ݘVѬ峿ۅ8Dh$~mm啠~'\ET&	 a}rMKL0u%HYL
l=`Υ3k[؝Y}$ ss8?~IXKda<==mL[RҠsHBR/*`JfUzA)'0JkArvp#e-{]U
Z`#2Ϡv~.#l7"D=&t^-Q_9Mi
uԒn{Zn!U%r3J;Q׼Di@PNg]&;yw|9B*.L=Ij-)/]'g^U0#0b=0
	*H
0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUS0
121023180003Z
141023180003Z0`10
	&,d1306910UMike Carlson10	UIT10U

Bay Photo Lab10	UUS0"0
	*H
0
<ȼ^|=e9KtФ-jI_	%[߲'O%3;=*n((RT	•͐C/\WU@HCjrIU-iE˼|paҨm-4݈amƵbK$"UEkEzd
w.
wG u:B'9!?tdk%%̞N.8C1ަί[
BjF0){C9&pXnĉZuX")3zsS\\D:L׏1Q}1Gzz(d#V3fRoш^CLfQ@S/StX
d5Y3M0ՙQ5ō;pIdV]&d#26zsgM}r#iМ|3)md:}뚁00R+F0D0B+06http://bayca.bayhoto.local/ejbca/publicweb/status/ocsp0U(}awJ״(#0U00U#0-rfbb,v0U00U%0
+0U0mike@bayphoto.com0
	*H
9|&V,*Hd	ƏA~6fFg'^y
I'yy,v}Z	@ᔘ7\F5QA37*LT4VStTe .Dӧ=n}=L\E	{
z7kYs#RO}E`OnL'1M0`Dۋ
rvVuX?s=	+O0:yE?BA̡5|Ʀpp*<FLA36k덝j9b=&)KJSmʐXo@g;V4@ujkX9	@Wh#nl\Y)A
rFGjqtvhu.ճK)L}@41AKz&ȴztÈ6͢j=0*+@;xnc-
WƣLG9X)=
y%]Q@BW
,Άut00MiC0
	*H
0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUS0
121023175745Z
141023175745Z0`10
	&,d1306910UMike Carlson10	UIT10U

Bay Photo Lab10	UUS0"0
	*H
0
@vɌAļVAW5:eh$n>b%k7Pwޡ=^CBv2ULLqn6+>A:P#=ѕ[8Z<|&wb(x椉
iҒx9H?~Ɔ-y]jN崡1geAˇwH4w?h!/^Pؕfa5-+%<*/+`ZBCƀno|6'zoe!)@H藱$zѩ+
SXDz(~Bݬe?V
\j;.P,銉[JݦkjY*nȡ5]
hlkz3.Wme/tɧ#	8L%
Ũ%zp	_p)ڜ(C=MYe3S>Tfρ=@	]ڑav&0ۗ;.j'Yk_00R+F0D0B+06http://bayca.bayhoto.local/ejbca/publicweb/status/ocsp0UFO+Rdb`?60U00U#0-rfbb,v0U00http://bayca.bayhoto.local/ejbca/publicweb/webdist/certdist?cmd=crl&issuer=CN=Bay%20Photo%20People%20CA,O=Bay%20Photo%20Lab,ST=California,C=US0U0U%0++0U0mike@bayphoto.com0
	*H
/ungfsy@KLw.cM&6?-Y4++IJYD	C£S_2$eڏPU((̖S~aM0ri~jk2Ւ[n9rn&Bz(MݼIܪ*ȱImu5lr[Q`3͈;l{Z07h$>at)qo\]pJW7*[c%

y1FB)p2͞[~=?!Wd9XY5.bOKUDV[Z98E
^X9n<Hi@C?H+jlۗc݌&yqQ<Ii/
ɣ*B!f<.Re-=Y*?-4;|vj1@+Iܑ=J7%'jMmrSM@GV|:C'ݮ_Lkt61F0B0d0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUSMiC0	+0	*H
	1	*H
0	*H
	1
140602182442Z0#	*H
	1-?`0l	*H
	1_0]0	`He*0	`He0
*H
0*H
0
*H
@0+0
*H
(0s	+71f0d0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUSb=0u*H
	1fd0X10UBay Photo People CA10U

Bay Photo Lab10U
California10	UUSb=0
	*H
BPߩv_%]Po=n<
5`d̈a7(+DXMJ9i6$ZQ2caOsyv!♇X-؈UݥݒͭM6+ĝ4T %@arwa?lGrwTb/D+~޷K7:.t}B%mp_TJ@kN:5RƗ	
%i9
_˧zPN]ңc	J2%NuAP5j{6iC]iLFvPb@Zs4t.qYY0c~X{vC
@{σA_'~`O:Ȇ8ZjY!D`K	Q umZhH#}BYisʒZ}<5ƃF"5{6#|껙HOyeG;B>?>1Yi=c)}y6

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?538CC16A.6060207>