Date: Wed, 22 Aug 2012 15:27:53 -0400 From: Roger Hammerstein <cheeky.m@live.com> To: <avg@freebsd.org> Cc: freebsd-fs@freebsd.org Subject: RE: panic while zfs scrubbing Message-ID: <BAY170-W1147AC6AE266A62ED84BE53F9BF0@phx.gbl> In-Reply-To: <5034DA84.8050507@FreeBSD.org> References: <BAY170-W8668C02B4DAF69B54EE657F9B80@phx.gbl>, <5034DA84.8050507@FreeBSD.org>
next in thread | previous in thread | raw e-mail | index | archive | help
Thank you for the reply.
> Not sure what triggers this problem but it looks like zio is issued for a
> block-pointer with no valid DVA. It's either a result of some logical bu=
g in ZFS
> code or some severe on-disk corruption.
>=20
> > #7 0xffffffff816818cf in vdev_mirror_io_start (zio=3D0xfffffe0037e5e00=
0)
> > at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/comm=
on/fs/zfs/vdev_mirror.c:89
>=20
> Could you please print *zio and *zio->io_bp in this frame?
> It might also be good idea to report this issue to zfs-discuss@opensolari=
s.org.
/var/crash # kgdb /boot/kernel/kernel ./vmcore.10
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation=2C Inc.
GDB is free software=2C covered by the GNU General Public License=2C and yo=
u are
welcome to change it and/or distribute copies of it under certain condition=
s.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "amd64-marcel-freebsd"...
Unread portion of the kernel message buffer:
Fatal trap 18: integer divide fault while in kernel mode
cpuid =3D 5=3B apic id =3D 05
instruction pointer =3D 0x20:0xffffffff81674a14
stack pointer =3D 0x28:0xffffff810c3d4520
frame pointer =3D 0x28:0xffffff810c3d4540
code segment =3D base 0x0=2C limit 0xfffff=2C type 0x1b
=3D DPL 0=2C pres 1=2C long 1=2C def32 0=2C gran 1
processor eflags =3D interrupt enabled=2C resume=2C IOPL =3D 0
current process =3D 9480 (txg_thread_enter)
trap number =3D 18
panic: integer divide fault
cpuid =3D 5
KDB: stack backtrace:
#0 0xffffffff80920346 at kdb_backtrace+0x66
#1 0xffffffff808ea35e at panic+0x1ce
#2 0xffffffff80bd7a30 at trap_fatal+0x290
#3 0xffffffff80bd80c5 at trap+0x105
#4 0xffffffff80bc295f at calltrap+0x8
#5 0xffffffff816818cf at vdev_mirror_io_start+0x2bf
#6 0xffffffff81699542 at zio_vdev_io_start+0x232
#7 0xffffffff81698fe3 at zio_execute+0xc3
#8 0xffffffff8165ea1c at dsl_scan_scrub_cb+0x3ec
#9 0xffffffff8165fe14 at dsl_scan_visitbp+0x534
#10 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9
#11 0xffffffff81660c84 at dsl_scan_visitdnode+0x84
#12 0xffffffff81660070 at dsl_scan_visitbp+0x790
#13 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9
#14 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9
#15 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9
#16 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9
#17 0xffffffff8165fd99 at dsl_scan_visitbp+0x4b9
Uptime: 1h51m55s
Dumping 614 out of 3818 MB:..3%..11%..21%..32%..42%..53%..63%..71%..81%..92=
%
Reading symbols from /boot/kernel/zfs.ko...Reading symbols from /boot/kerne=
l/zfs.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/zfs.ko
Reading symbols from /boot/kernel/opensolaris.ko...Reading symbols from /bo=
ot/kernel/opensolaris.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/opensolaris.ko
#0 doadump (textdump=3DVariable "textdump" is not available.
) at pcpu.h:224
224 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) frame 7
#7 0xffffffff816818cf in vdev_mirror_io_start (zio=3D0xfffffe0037e5e000)
at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/f=
s/zfs/vdev_mirror.c:89
89 mm->mm_preferred =3D spa_get_random(c)=3B
(kgdb) p *zio
$1 =3D {io_bookmark =3D {zb_objset =3D 21=2C zb_object =3D 291417=2C zb_lev=
el =3D 0=2C zb_blkid =3D 41}=2C io_prop =3D {
zp_checksum =3D ZIO_CHECKSUM_INHERIT=2C zp_compress =3D ZIO_COMPRESS_IN=
HERIT=2C zp_type =3D DMU_OT_NONE=2C
zp_level =3D 0 '\0'=2C zp_copies =3D 0 '\0'=2C zp_dedup =3D 0 '\0'=2C z=
p_dedup_verify =3D 0 '\0'}=2C
io_type =3D ZIO_TYPE_READ=2C io_child_type =3D ZIO_CHILD_LOGICAL=2C io_cm=
d =3D 0=2C io_priority =3D 20 '\024'=2C
io_reexecute =3D 0 '\0'=2C io_state =3D "\001"=2C io_txg =3D 223=2C io_sp=
a =3D 0xfffffe0008161000=2C
io_bp =3D 0xfffffe0037e5e060=2C io_bp_override =3D 0x0=2C io_bp_copy =3D =
{blk_dva =3D {{dva_word =3D {0=2C 0}}=2C {
dva_word =3D {0=2C 0}}=2C {dva_word =3D {0=2C 0}}}=2C blk_prop =3D =
0=2C blk_pad =3D {0=2C 223}=2C blk_phys_birth =3D 223=2C
blk_birth =3D 250=2C blk_fill =3D 105=2C blk_cksum =3D {zc_word =3D {63=
=2C 125=2C 202=2C 251}}}=2C io_parent_list =3D {
list_size =3D 48=2C list_offset =3D 16=2C list_head =3D {list_next =3D =
0xfffffe0037ea28b0=2C
list_prev =3D 0xfffffe0037ea28b0}}=2C io_child_list =3D {list_size =
=3D 48=2C list_offset =3D 32=2C list_head =3D {
list_next =3D 0xfffffe0037e5e110=2C list_prev =3D 0xfffffe0037e5e110}=
}=2C io_walk_link =3D 0x0=2C
io_logical =3D 0xfffffe0037e5e000=2C io_transform_stack =3D 0x0=2C io_rea=
dy =3D 0=2C
io_done =3D 0xffffffff8165e130 <dsl_scan_scrub_done>=2C io_private =3D 0x=
0=2C io_prev_space_delta =3D 0=2C
io_bp_orig =3D {blk_dva =3D {{dva_word =3D {0=2C 0}}=2C {dva_word =3D {0=
=2C 0}}=2C {dva_word =3D {0=2C 0}}}=2C blk_prop =3D 0=2C
blk_pad =3D {0=2C 223}=2C blk_phys_birth =3D 223=2C blk_birth =3D 250=
=2C blk_fill =3D 105=2C blk_cksum =3D {zc_word =3D {63=2C
125=2C 202=2C 251}}}=2C io_data =3D 0xfffffe003704e200=2C io_orig_d=
ata =3D 0xfffffe003704e200=2C io_size =3D 512=2C
io_orig_size =3D 512=2C io_vd =3D 0x0=2C io_vsd =3D 0x0=2C io_vsd_ops =3D=
0x0=2C io_offset =3D 0=2C io_deadline =3D 0=2C
io_offset_node =3D {avl_child =3D {0x0=2C 0x0}=2C avl_pcb =3D 0}=2C io_de=
adline_node =3D {avl_child =3D {0x0=2C 0x0}=2C
avl_pcb =3D 0}=2C io_vdev_tree =3D 0x0=2C io_flags =3D 2097264=2C io_st=
age =3D ZIO_STAGE_VDEV_IO_START=2C
io_pipeline =3D 2064386=2C io_orig_flags =3D 2097264=2C io_orig_stage =3D=
ZIO_STAGE_OPEN=2C
io_orig_pipeline =3D 2064386=2C io_error =3D 0=2C io_child_error =3D {0=
=2C 0=2C 0=2C 0}=2C io_children =3D {{0=2C 0}=2C {0=2C 0}=2C {
0=2C 0}=2C {0=2C 0}}=2C io_child_count =3D 0=2C io_parent_count =3D 1=
=2C io_stall =3D 0x0=2C io_gang_leader =3D 0x0=2C
io_gang_tree =3D 0x0=2C io_executor =3D 0xfffffe0015e9d470=2C io_waiter =
=3D 0x0=2C io_lock =3D {lock_object =3D {
lo_name =3D 0xffffffff81725cee "zio->io_lock"=2C lo_flags =3D 4096000=
0=2C lo_data =3D 0=2C lo_witness =3D 0x0}=2C
sx_lock =3D 1}=2C io_cv =3D {cv_description =3D 0xffffffff81725cfd "zio=
->io_cv)"=2C cv_waiters =3D 0}=2C
io_cksum_report =3D 0x0=2C io_ena =3D 0=2C io_task =3D {ost_task =3D {ta_=
link =3D {stqe_next =3D 0x0}=2C ta_pending =3D 0=2C
ta_priority =3D 0=2C ta_func =3D 0=2C ta_context =3D 0x0}=2C ost_func=
=3D 0=2C ost_arg =3D 0x0}}
(kgdb)
(kgdb) p *zio->io_bp
$2 =3D {blk_dva =3D {{dva_word =3D {0=2C 0}}=2C {dva_word =3D {0=2C 0}}=2C =
{dva_word =3D {0=2C 0}}}=2C blk_prop =3D 0=2C blk_pad =3D {
0=2C 223}=2C blk_phys_birth =3D 223=2C blk_birth =3D 250=2C blk_fill =
=3D 105=2C blk_cksum =3D {zc_word =3D {63=2C 125=2C 202=2C
251}}}
(kgdb)
> > #8 0xffffffff81699542 in zio_vdev_io_start (zio=3D0xfffffe0037e5e000)
> > at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/comm=
on/fs/zfs/zio.c:2305
> > #9 0xffffffff81698fe3 in zio_execute (zio=3D0xfffffe0037e5e000)
> > at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/comm=
on/fs/zfs/zio.c:1196
> > #10 0xffffffff8165ea1c in dsl_scan_scrub_cb (dp=3D0xffffff810c3d4538=2C=
=20
> > bp=3D0xffffff8003c53480=2C zb=3D0xffffff810c3d4970)
> > at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/comm=
on/fs/zfs/dsl_scan.c:1737
>=20
> And *bp and *scn here too.
(kgdb) frame 10
#10 0xffffffff8165ea1c in dsl_scan_scrub_cb (dp=3D0xffffff810c3d4538=2C bp=
=3D0xffffff8003c53480=2C
zb=3D0xffffff810c3d4970)
at /usr/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common/f=
s/zfs/dsl_scan.c:1737
1737 zio_nowait(zio_read(NULL=2C spa=2C bp=2C data=2C si=
ze=2C
(kgdb) p *bp
$3 =3D {blk_dva =3D {{dva_word =3D {0=2C 0}}=2C {dva_word =3D {0=2C 0}}=2C =
{dva_word =3D {0=2C 0}}}=2C blk_prop =3D 0=2C blk_pad =3D {
0=2C 223}=2C blk_phys_birth =3D 223=2C blk_birth =3D 250=2C blk_fill =
=3D 105=2C blk_cksum =3D {zc_word =3D {63=2C 125=2C 202=2C
251}}}
(kgdb) p *scn
Cannot access memory at address 0x20
(kgdb)
=
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?BAY170-W1147AC6AE266A62ED84BE53F9BF0>