Date: Wed, 10 Mar 2010 16:12:36 +0100 From: Borja Marcos <borjam@sarenet.es> To: Pawel Jakub Dawidek <pjd@FreeBSD.org> Cc: freebsd-fs@freebsd.org, FreeBSD Stable <freebsd-stable@freebsd.org> Subject: Re: Many processes stuck in zfs Message-ID: <E04F91AA-B2C4-4166-A24A-74F1BEF01519@sarenet.es> In-Reply-To: <20100310110202.GA1715@garage.freebsd.pl> References: <864468D4-DCE9-493B-9280-00E5FAB2A05C@lassitu.de> <20100309122954.GE3155@garage.freebsd.pl> <EC9BC6B4-8D0E-4FE3-852F-0E3A24569D33@sarenet.es> <20100309125815.GF3155@garage.freebsd.pl> <CB854F58-03AF-46DD-8153-85FA96037C21@sarenet.es> <BFF1E2D6-B48A-4A5E-ACEE-8577FDB07820@sarenet.es> <20100310110202.GA1715@garage.freebsd.pl>
next in thread | previous in thread | raw e-mail | index | archive | help
=09
On Mar 10, 2010, at 12:02 PM, Pawel Jakub Dawidek wrote:
> Once the deadlock occur, enter DDB and send me the output of:
>=20
> ps
> show alllocks
> show lockedvnods
> show allchains
> alltrace
(Again, crossposted to -fs, ZFS related)
Previous one was a panic when performing the test with several tar jobs =
running in parallel.
Now this is a capture of the deadlock itself, instead of a panic. (I =
called panic from the debugger to generate a dump)
ddb.txt:
# cat ddb.txt
db> set $lines=3D0
db> ps
pid ppid pgrp uid state wmesg wchan cmd
1821 1385 1821 0 R+ CPU 0 sysctl
1820 1818 1818 0 S tx->tx_s 0xffffff00027da650 zfs
1818 1816 1818 0 Ss pause 0xffffff0002e33960 csh
1816 1284 1816 0 Ss select 0xffffff001c6d3cc0 sshd
1699 1697 1697 0 S+ zfsvfs-> 0xffffff0002893210 bsdtar
1698 1697 1697 0 S+ pipdwt 0xffffff00025d62d8 bsdtar
1697 1385 1697 0 S+ wait 0xffffff0002d47000 sh
1385 1384 1385 0 R+ csh
1384 1381 1384 1001 S+ wait 0xffffff0002e33460 su
1381 1380 1381 1001 Ss+ pause 0xffffff0002e36500 csh
1380 1377 1377 1001 R sshd
1377 1284 1377 0 Ss sbwait 0xffffff0002c763ec sshd
1376 1 1376 0 Ss+ ttyin 0xffffff00024daca8 getty
1375 1 1375 0 Ss+ ttyin 0xffffff00024dc0a8 getty
1374 1 1374 0 Ss+ ttyin 0xffffff00024cdca8 getty
1373 1 1373 0 Ss+ ttyin 0xffffff00024c94a8 getty
1372 1 1372 0 Ss+ ttyin 0xffffff00024dd4a8 getty
1371 1 1371 0 Ss+ ttyin 0xffffff00024c28a8 getty
1370 1 1370 0 Ss+ ttyin 0xffffff00024db4a8 getty
1369 1 1369 0 Ss+ ttyin 0xffffff00024dc8a8 getty
1305 1 1305 0 Ss nanslp 0xffffffff80c4d168 cron
1298 1 1298 25 Ss pause 0xffffff000257d500 sendmail
1292 1 1292 0 Ss select 0xffffff000287f5c0 sendmail
1284 1 1284 0 Ss select 0xffffff000287f940 sshd
1005 1 1005 0 Ss select 0xffffff0002c6e2c0 syslogd
827 1 827 0 Ss select 0xffffff000296f440 devd
20 0 0 0 SL flowclea 0xffffffff80e0e950 =
[flowcleaner]
19 0 0 0 SL sdflush 0xffffffff80e1d8b8 =
[softdepflush]
18 0 0 0 SL rrl->rr_ 0xffffff0002893110 [syncer]
17 0 0 0 SL zfsvfs-> 0xffffff0002893210 [vnlru]
16 0 0 0 SL psleep 0xffffffff80e0e1c8 [bufdaemon]
15 0 0 0 SL pgzero 0xffffffff80e1f34c [pagezero]
14 0 0 0 SL psleep 0xffffffff80e1e6e8 [vmdaemon]
9 0 0 0 SL psleep 0xffffffff80e1e6ac =
[pagedaemon]
8 0 0 0 SL ccb_scan 0xffffffff80c16060 [xpt_thrd]
7 0 0 0 SL (threaded) zfskern
100085 D tx->tx_q 0xffffff00027da670 =
[txg_thread_enter]
100084 D tx->tx_c 0xffffff0002599d20 =
[txg_thread_enter]
100082 D vgeom:io 0xffffff00026acd90 [vdev da2]
100036 D l2arc_fe 0xffffffff81150620 =
[l2arc_feed_thread]
100035 D arc_recl 0xffffffff81147fe0 =
[arc_reclaim_thread]
6 0 0 0 SL - 0xffffff000244e648 [fdc0]
5 0 0 0 SL idle 0xffffff800023c000 =
[mpt_recovery0]
13 0 0 0 SL - 0xffffffff80c4ce44 [yarrow]
4 0 0 0 SL - 0xffffffff80c49648 [g_down]
3 0 0 0 SL - 0xffffffff80c49640 [g_up]
2 0 0 0 SL - 0xffffffff80c49630 [g_event]
12 0 0 0 WL (threaded) intr
100033 I [swi0: uart =
uart]
100032 I [irq7: =
ppc0]
100031 I [irq12: =
psm0]
100030 I [irq1: =
atkbd0]
100024 I [irq17: em2 =
mpt0]
100023 I [irq15: =
ata1]
100022 I [irq14: =
ata0]
100021 I [irq9: =
acpi0]
100019 I [swi5: +]
100018 I [swi2: =
cambio]
100013 I [swi6: task =
queue]
100012 I [swi6: =
Giant taskq]
100006 I [swi1: =
netisr 0]
100005 I [swi4: =
clock]
100004 I [swi3: vm]
11 0 0 0 RL [idle: =
cpu0]
1 0 1 0 SLs wait 0xffffff00021e18c0 [init]
10 0 0 0 SL audit_wo 0xffffffff80e1cc10 [audit]
0 0 0 0 SLs (threaded) kernel
100087 D - 0xffffff000287c180 [zil_clean]
100086 D - 0xffffff000287b200 [zil_clean]
100083 D - 0xffffff0002640480 =
[zfs_vn_rele_taskq]
100081 D - 0xffffff00026acc00 [spa_zio]
100080 D - 0xffffff00026acb80 [spa_zio]
100079 D - 0xffffff00026acb00 [spa_zio]
100078 D - 0xffffff00026aca80 [spa_zio]
100077 D - 0xffffff00026aca00 [spa_zio]
100076 D - 0xffffff00026ac980 [spa_zio]
100075 D - 0xffffff00026ac900 [spa_zio]
100074 D - 0xffffff00026ac880 [spa_zio_7]
100073 D - 0xffffff00026ac880 [spa_zio_6]
100072 D - 0xffffff00026ac880 [spa_zio_5]
100071 D - 0xffffff00026ac880 [spa_zio_4]
100070 D - 0xffffff00026ac880 [spa_zio_3]
100069 D - 0xffffff00026ac880 [spa_zio_2]
100068 D - 0xffffff00026ac880 [spa_zio_1]
100067 D - 0xffffff00026ac880 [spa_zio_0]
100066 D - 0xffffff00026ac800 [spa_zio_7]
100065 D - 0xffffff00026ac800 [spa_zio_6]
100064 D - 0xffffff00026ac800 [spa_zio_5]
100063 D - 0xffffff00026ac800 [spa_zio_4]
100062 D - 0xffffff00026ac800 [spa_zio_3]
100061 D - 0xffffff00026ac800 [spa_zio_2]
100060 D - 0xffffff00026ac800 [spa_zio_1]
100059 D - 0xffffff00026ac800 [spa_zio_0]
100058 D - 0xffffff00026ac780 [spa_zio]
100057 D - 0xffffff00026ac700 [spa_zio]
100056 D - 0xffffff00026ac680 [spa_zio]
100029 D - 0xffffff0002441980 [em3 taskq]
100028 D - 0xffffff0002440d80 [em2 taskq]
100027 D - 0xffffff0002436200 [em1 taskq]
100026 D - 0xffffff000242fb80 [em0 taskq]
100020 D - 0xffffff0002358600 [thread =
taskq]
100017 D - 0xffffff0002316b80 =
[acpi_task_2]
100016 D - 0xffffff0002316b80 =
[acpi_task_1]
100015 D - 0xffffff0002316b80 =
[acpi_task_0]
100014 D - 0xffffff0002316c00 [kqueue =
taskq]
100010 D - 0xffffff00021e0180 [firmware =
taskq]
100000 D sched 0xffffffff80c49740 [swapper]
db> show alllocks
Process 1821 (sysctl) thread 0xffffff00025b0ae0 (100050)
Process 1820 (zfs) thread 0xffffff0002ca4000 (100105)
Process 1699 (bsdtar) thread 0xffffff000262dae0 (100093)
Process 1377 (sshd) thread 0xffffff0002642740 (100090)
Process 18 (syncer) thread 0xffffff000250d3a0 (100043)
Process 17 (vnlru) thread 0xffffff000250d740 (100042)
db> show lockedvnods
Locked vnodes
db> show allchains
db> alltrace
Tracing command sysctl pid 1821 tid 100050 td 0xffffff00025b0ae0
kdb_enter() at kdb_enter+0x3d
kdb_sysctl_enter() at kdb_sysctl_enter+0x89
sysctl_root() at sysctl_root+0x113
userland_sysctl() at userland_sysctl+0x158
__sysctl() at __sysctl+0xaa
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (202, FreeBSD ELF64, __sysctl), rip =3D 0x80073cfac, rsp =3D =
0x7fffffffe298, rbp =3D 0x7fffffffe36c ---
Tracing command zfs pid 1820 tid 100105 td 0xffffff0002ca4000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
txg_wait_synced() at txg_wait_synced+0x98
zfsvfs_teardown() at zfsvfs_teardown+0x1f6
zfs_suspend_fs() at zfs_suspend_fs+0x2b
zfs_ioc_recv() at zfs_ioc_recv+0x28b
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffff8e98, rbp =3D 0x7fffffff9bd0 ---
Tracing command csh pid 1818 tid 100102 td 0xffffff0002ca4ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip =3D 0x80094d19c, rsp =3D =
0x7fffffffe578, rbp =3D 0x800c0f500 ---
Tracing command sshd pid 1816 tid 100098 td 0xffffff0002d2bae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip =3D 0x8013d7cfc, rsp =3D =
0x7fffffffdcb8, rbp =3D 0x7fffffffdd40 ---
Tracing command bsdtar pid 1699 tid 100093 td 0xffffff000262dae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sx_slock_hard() at _sx_slock_hard+0x1b7
_sx_slock() at _sx_slock+0xc1
zfs_freebsd_reclaim() at zfs_freebsd_reclaim+0x63
VOP_RECLAIM_APV() at VOP_RECLAIM_APV+0xb5
vgonel() at vgonel+0x119
vnlru_free() at vnlru_free+0x345
getnewvnode() at getnewvnode+0x24f
zfs_znode_cache_constructor() at zfs_znode_cache_constructor+0x43
zfs_znode_alloc() at zfs_znode_alloc+0x38
zfs_mknode() at zfs_mknode+0x259
zfs_freebsd_create() at zfs_freebsd_create+0x661
VOP_CREATE_APV() at VOP_CREATE_APV+0xb3
vn_open_cred() at vn_open_cred+0x473
kern_openat() at kern_openat+0x179
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (5, FreeBSD ELF64, open), rip =3D 0x800e35b4c, rsp =3D =
0x7fffffffe748, rbp =3D 0x1a4 ---
Tracing command bsdtar pid 1698 tid 100107 td 0xffffff0002c90740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
pipe_write() at pipe_write+0x13fc
dofilewrite() at dofilewrite+0x85
kern_writev() at kern_writev+0x60
write() at write+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip =3D 0x800e44d5c, rsp =3D =
0x7fffffffe818, rbp =3D 0x801004140 ---
Tracing command sh pid 1697 tid 100091 td 0xffffff00026423a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip =3D 0x80093554c, rsp =3D =
0x7fffffffe8a8, rbp =3D 0x6a1 ---
Tracing command csh pid 1385 tid 100104 td 0xffffff0002ca43a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
fork1() at fork1+0xfe2
vfork() at vfork+0x1c
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (66, FreeBSD ELF64, vfork), rip =3D 0x80094f31d, rsp =3D =
0x7fffffffe6c0, rbp =3D 0 ---
Tracing command su pid 1384 tid 100103 td 0xffffff0002ca4740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip =3D 0x8009e854c, rsp =3D =
0x7fffffffe598, rbp =3D 0x569 ---
Tracing command csh pid 1381 tid 100100 td 0xffffff0002d2b3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip =3D 0x80094d19c, rsp =3D =
0x7fffffffe558, rbp =3D 0x800c0f900 ---
Tracing command sshd pid 1380 tid 100099 td 0xffffff0002d2b740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip =3D 0x8013d7cfc, rsp =3D =
0x7fffffffdcb8, rbp =3D 0x7fffffffdd40 ---
Tracing command sshd pid 1377 tid 100090 td 0xffffff0002642740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
soreceive_generic() at soreceive_generic+0xf99
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x8013d7d7c, rsp =3D =
0x7fffffffdcd8, rbp =3D 0 ---
Tracing command getty pid 1376 tid 100046 td 0xffffff00025b1ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1375 tid 100095 td 0xffffff000262d3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1374 tid 100096 td 0xffffff0002cd43a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1373 tid 100097 td 0xffffff0002cd4000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1372 tid 100092 td 0xffffff0002642000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1371 tid 100094 td 0xffffff000262d740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1370 tid 100048 td 0xffffff00025b13a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command getty pid 1369 tid 100054 td 0xffffff0002510ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
tty_wait() at tty_wait+0x48
ttydisc_read() at ttydisc_read+0x2f1
ttydev_read() at ttydev_read+0xab
devfs_read_f() at devfs_read_f+0x86
dofileread() at dofileread+0xa1
kern_readv() at kern_readv+0x60
read() at read+0x55
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (3, FreeBSD ELF64, read), rip =3D 0x80084ed7c, rsp =3D =
0x7fffffffecc8, rbp =3D 0 ---
Tracing command cron pid 1305 tid 100052 td 0xffffff00025b03a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_sleep() at _sleep+0x23f
kern_nanosleep() at kern_nanosleep+0x118
nanosleep() at nanosleep+0x6e
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (240, FreeBSD ELF64, nanosleep), rip =3D 0x80093efac, rsp =3D =
0x7fffffffeb28, rbp =3D 0x3c ---
Tracing command sendmail pid 1298 tid 100055 td 0xffffff0002510740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_sigsuspend() at kern_sigsuspend+0x9a
sigsuspend() at sigsuspend+0x34
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (4, FreeBSD ELF64, write), rip =3D 0x800d3a19c, rsp =3D =
0x7fffffffcd68, rbp =3D 0x1 ---
Tracing command sendmail pid 1292 tid 100053 td 0xffffff00025b0000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_timedwait_sig() at sleepq_timedwait_sig+0x19
_cv_timedwait_sig() at _cv_timedwait_sig+0x18c
seltdwait() at seltdwait+0x56
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip =3D 0x800dd8cfc, rsp =3D =
0x7fffffffc1e8, rbp =3D 0x7fffffffc280 ---
Tracing command sshd pid 1284 tid 100047 td 0xffffff00025b1740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip =3D 0x8013d7cfc, rsp =3D =
0x7fffffffddd8, rbp =3D 0x2 ---
Tracing command syslogd pid 1005 tid 100051 td 0xffffff00025b0740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip =3D 0x800851cfc, rsp =3D =
0x7fffffffe118, rbp =3D 0x800a310c8 ---
Tracing command devd pid 827 tid 100049 td 0xffffff00025b1000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_cv_wait_sig() at _cv_wait_sig+0x17e
seltdwait() at seltdwait+0xac
kern_select() at kern_select+0x620
select() at select+0x5d
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (93, FreeBSD ELF64, select), rip =3D 0x440ecc, rsp =3D =
0x7fffffffe898, rbp =3D 0x7fffffffe8b0 ---
Tracing command flowcleaner pid 20 tid 100045 td 0xffffff0002485ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
flowtable_cleaner() at flowtable_cleaner+0xde
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800018cd30, rbp =3D 0 ---
Tracing command softdepflush pid 19 tid 100044 td 0xffffff000250d000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
softdep_flush() at softdep_flush+0x259
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000187d30, rbp =3D 0 ---
Tracing command syncer pid 18 tid 100043 td 0xffffff000250d3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
rrw_enter() at rrw_enter+0x135
zfs_sync() at zfs_sync+0x63
sync_fsync() at sync_fsync+0x13a
VOP_FSYNC_APV() at VOP_FSYNC_APV+0xb5
sync_vnode() at sync_vnode+0x157
sched_sync() at sched_sync+0x1cc
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000182d30, rbp =3D 0 ---
Tracing command vnlru pid 17 tid 100042 td 0xffffff000250d740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sx_slock_hard() at _sx_slock_hard+0x1b7
_sx_slock() at _sx_slock+0xc1
zfs_freebsd_reclaim() at zfs_freebsd_reclaim+0x63
VOP_RECLAIM_APV() at VOP_RECLAIM_APV+0xb5
vgonel() at vgonel+0x119
vnlru_free() at vnlru_free+0x345
vnlru_proc() at vnlru_proc+0x58e
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800017dd30, rbp =3D 0 ---
Tracing command bufdaemon pid 16 tid 100041 td 0xffffff000250dae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
buf_daemon() at buf_daemon+0x14a
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000178d30, rbp =3D 0 ---
Tracing command pagezero pid 15 tid 100040 td 0xffffff000250e000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
vm_pagezero() at vm_pagezero+0x73
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000173d30, rbp =3D 0 ---
Tracing command vmdaemon pid 14 tid 100039 td 0xffffff000250e3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
vm_daemon() at vm_daemon+0x4d
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800016ed30, rbp =3D 0 ---
Tracing command pagedaemon pid 9 tid 100038 td 0xffffff000250e740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
vm_pageout() at vm_pageout+0x855
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000169d30, rbp =3D 0 ---
Tracing command xpt_thrd pid 8 tid 100037 td 0xffffff000250eae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
xpt_scanner_thread() at xpt_scanner_thread+0xd5
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000164d30, rbp =3D 0 ---
Tracing command zfskern pid 7 tid 100085 td 0xffffff0002643ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
txg_thread_wait() at txg_thread_wait+0x5b
txg_sync_thread() at txg_sync_thread+0x1d0
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
Tracing command zfskern pid 7 tid 100084 td 0xffffff0002645000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
txg_quiesce_thread() at txg_quiesce_thread+0x22b
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012413d30, rbp =3D 0 ---
Tracing command zfskern pid 7 tid 100082 td 0xffffff000263dae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
vdev_geom_worker() at vdev_geom_worker+0xf3
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123ffd30, rbp =3D 0 ---
Tracing command zfskern pid 7 tid 100036 td 0xffffff0002510000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
l2arc_feed_thread() at l2arc_feed_thread+0x162
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800015fd30, rbp =3D 0 ---
Tracing command zfskern pid 7 tid 100035 td 0xffffff0002356740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_cv_timedwait() at _cv_timedwait+0x18c
arc_reclaim_thread() at arc_reclaim_thread+0x2b7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800015ad30, rbp =3D 0 ---
Tracing command fdc0 pid 6 tid 100034 td 0xffffff0002356ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
fdc_thread() at fdc_thread+0x774
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000155d30, rbp =3D 0 ---
Tracing command mpt_recovery0 pid 5 tid 100025 td 0xffffff0002353ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
mpt_recovery_thread() at mpt_recovery_thread+0x251
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800011ed30, rbp =3D 0 ---
Tracing command yarrow pid 13 tid 100011 td 0xffffff00021e53a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
random_kthread() at random_kthread+0x1ad
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800004cd30, rbp =3D 0 ---
Tracing command g_down pid 4 tid 100009 td 0xffffff00021e5ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
g_io_schedule_down() at g_io_schedule_down+0x236
g_down_procbody() at g_down_procbody+0x6f
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000042d30, rbp =3D 0 ---
Tracing command g_up pid 3 tid 100008 td 0xffffff00021f3000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
g_io_schedule_up() at g_io_schedule_up+0x133
g_up_procbody() at g_up_procbody+0x6f
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800003dd30, rbp =3D 0 ---
Tracing command g_event pid 2 tid 100007 td 0xffffff00021f33a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
g_event_procbody() at g_event_procbody+0xa1
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000038d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100033 td 0xffffff0002483000
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100032 td 0xffffff00024833a0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100031 td 0xffffff0002483740
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100030 td 0xffffff0002483ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000137d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100024 td 0xffffff0002354000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80000d7d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100023 td 0xffffff00023543a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80000d2d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100022 td 0xffffff0002354740
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100021 td 0xffffff0002354ae0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100019 td 0xffffff00023563a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000078d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100018 td 0xffffff00021f3ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000073d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100013 td 0xffffff0002353000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800005ad30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100012 td 0xffffff00023533a0
fork_trampoline() at fork_trampoline
Tracing command intr pid 12 tid 100006 td 0xffffff00021f3740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000033d30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100005 td 0xffffff00021e4000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
ithread_loop() at ithread_loop+0x246
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800002ed30, rbp =3D 0 ---
Tracing command intr pid 12 tid 100004 td 0xffffff00021e43a0
fork_trampoline() at fork_trampoline
Tracing command idle pid 11 tid 100003 td 0xffffff00021e4740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
critical_exit() at critical_exit+0xa3
intr_event_handle() at intr_event_handle+0xd7
intr_execute_handlers() at intr_execute_handlers+0x5f
lapic_handle_intr() at lapic_handle_intr+0x37
Xapic_isr1() at Xapic_isr1+0xa4
--- interrupt, rip =3D 0xffffffff808545f6, rsp =3D 0xffffff8000024b50, =
rbp =3D 0xffffff8000024b60 ---
acpi_cpu_c1() at acpi_cpu_c1+0x6
acpi_cpu_idle() at acpi_cpu_idle+0x1f6
sched_idletd() at sched_idletd+0x123
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000024d30, rbp =3D 0 ---
Tracing command init pid 1 tid 100002 td 0xffffff00021e4ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_catch_signals() at sleepq_catch_signals+0x2af
sleepq_wait_sig() at sleepq_wait_sig+0x16
_sleep() at _sleep+0x309
kern_wait() at kern_wait+0x3f7
wait4() at wait4+0x35
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (7, FreeBSD ELF64, wait4), rip =3D 0x40c86c, rsp =3D =
0x7fffffffe808, rbp =3D 0x401d40 ---
Tracing command audit pid 10 tid 100001 td 0xffffff00021e5000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_cv_wait() at _cv_wait+0x17a
audit_worker() at audit_worker+0x77
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800001ad30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100087 td 0xffffff00026433a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012421d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100086 td 0xffffff0002643740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff801241cd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100083 td 0xffffff000263d740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012404d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100081 td 0xffffff000263e000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123fad30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100080 td 0xffffff000263e3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123f5d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100079 td 0xffffff000263e740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123f0d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100078 td 0xffffff000263eae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123ebd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100077 td 0xffffff0002628000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123e6d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100076 td 0xffffff0002628740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123e1d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100075 td 0xffffff0002628ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123dcd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100074 td 0xffffff00025103a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123d7d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100073 td 0xffffff0002645740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123d2d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100072 td 0xffffff00026453a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123cdd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100071 td 0xffffff000262a000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80123c8d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100070 td 0xffffff000262a3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001ffd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100069 td 0xffffff000262a740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001fad30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100068 td 0xffffff000262aae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001d2d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100067 td 0xffffff000262b000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001f5d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100066 td 0xffffff000262b3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001f0d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100065 td 0xffffff000262b740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001ebd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100064 td 0xffffff000262bae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001e6d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100063 td 0xffffff000262c000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001e1d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100062 td 0xffffff000262c3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001dcd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100061 td 0xffffff000262c740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001d7d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100060 td 0xffffff000262cae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001cdd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100059 td 0xffffff000262d000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001c8d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100058 td 0xffffff000263d000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff80001c3d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100057 td 0xffffff000263d3a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012409d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100056 td 0xffffff00026283a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff801240ed30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100029 td 0xffffff0002485000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000132d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100028 td 0xffffff00024853a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800012dd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100027 td 0xffffff0002485740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000128d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100026 td 0xffffff0002353740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000123d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100020 td 0xffffff0002356000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800007dd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100017 td 0xffffff0002352000
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800006ed30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100016 td 0xffffff00023523a0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000069d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100015 td 0xffffff0002352740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
msleep_spin() at msleep_spin+0x209
taskqueue_thread_loop() at taskqueue_thread_loop+0x62
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000064d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100014 td 0xffffff0002352ae0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff800005fd30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100010 td 0xffffff00021e5740
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_wait() at sleepq_wait+0x4d
_sleep() at _sleep+0x357
taskqueue_thread_loop() at taskqueue_thread_loop+0xb7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8000047d30, rbp =3D 0 ---
Tracing command kernel pid 0 tid 100000 td 0xffffffff80c49ba0
sched_switch() at sched_switch+0x154
mi_switch() at mi_switch+0x21d
sleepq_switch() at sleepq_switch+0x123
sleepq_timedwait() at sleepq_timedwait+0x4d
_sleep() at _sleep+0x341
scheduler() at scheduler+0x29c
mi_startup() at mi_startup+0x59
btext() at btext+0x2c
db> textdump set
textdump set
db> help
ahd_dump ahd_in ahd_out ahd_pause ahd_sunit ahd_unpause
alltrace b break bt c call
capture continue d delete dhwatch dwatch
examine gdb halt hwatch kill match
next p panic print ps reboot
reset run s script scripts search
set show step t textdump thread
trace unscript until w watch watchdog
where write x
db> panic
#=20
msgbuf.txt:
#=20
#=20
# cat msgbuf.txt
2400-0x243f mem 0xd89a0000-0xd89bffff,0xd8930000-0xd893ffff irq 19 at =
device 5.0 on pci2
em3: Memory Access and/or Bus Master bits were not set!
em3: [FILTER]
em3: Ethernet address: 00:0c:29:11:4f:e9
pcib3: <ACPI PCI-PCI bridge> at device 21.0 on pci0
pci3: <ACPI PCI bus> on pcib3
pcib4: <ACPI PCI-PCI bridge> at device 21.1 on pci0
pci4: <ACPI PCI bus> on pcib4
pcib5: <ACPI PCI-PCI bridge> at device 21.2 on pci0
pci5: <ACPI PCI bus> on pcib5
pcib6: <ACPI PCI-PCI bridge> at device 21.3 on pci0
pci6: <ACPI PCI bus> on pcib6
pcib7: <ACPI PCI-PCI bridge> at device 21.4 on pci0
pci7: <ACPI PCI bus> on pcib7
pcib8: <ACPI PCI-PCI bridge> at device 21.5 on pci0
pci8: <ACPI PCI bus> on pcib8
pcib9: <ACPI PCI-PCI bridge> at device 21.6 on pci0
pci9: <ACPI PCI bus> on pcib9
pcib10: <ACPI PCI-PCI bridge> at device 21.7 on pci0
pci10: <ACPI PCI bus> on pcib10
pcib11: <ACPI PCI-PCI bridge> at device 22.0 on pci0
pci11: <ACPI PCI bus> on pcib11
pcib12: <ACPI PCI-PCI bridge> at device 22.1 on pci0
pci12: <ACPI PCI bus> on pcib12
pcib13: <ACPI PCI-PCI bridge> at device 22.2 on pci0
pci13: <ACPI PCI bus> on pcib13
pcib14: <ACPI PCI-PCI bridge> at device 22.3 on pci0
pci14: <ACPI PCI bus> on pcib14
pcib15: <ACPI PCI-PCI bridge> at device 22.4 on pci0
pci15: <ACPI PCI bus> on pcib15
pcib16: <ACPI PCI-PCI bridge> at device 22.5 on pci0
pci16: <ACPI PCI bus> on pcib16
pcib17: <ACPI PCI-PCI bridge> at device 22.6 on pci0
pci17: <ACPI PCI bus> on pcib17
pcib18: <ACPI PCI-PCI bridge> at device 22.7 on pci0
pci18: <ACPI PCI bus> on pcib18
pcib19: <ACPI PCI-PCI bridge> at device 23.0 on pci0
pci19: <ACPI PCI bus> on pcib19
pcib20: <ACPI PCI-PCI bridge> at device 23.1 on pci0
pci20: <ACPI PCI bus> on pcib20
pcib21: <ACPI PCI-PCI bridge> at device 23.2 on pci0
pci21: <ACPI PCI bus> on pcib21
pcib22: <ACPI PCI-PCI bridge> at device 23.3 on pci0
pci22: <ACPI PCI bus> on pcib22
pcib23: <ACPI PCI-PCI bridge> at device 23.4 on pci0
pci23: <ACPI PCI bus> on pcib23
pcib24: <ACPI PCI-PCI bridge> at device 23.5 on pci0
pci24: <ACPI PCI bus> on pcib24
pcib25: <ACPI PCI-PCI bridge> at device 23.6 on pci0
pci25: <ACPI PCI bus> on pcib25
pcib26: <ACPI PCI-PCI bridge> at device 23.7 on pci0
pci26: <ACPI PCI bus> on pcib26
pcib27: <ACPI PCI-PCI bridge> at device 24.0 on pci0
pci27: <ACPI PCI bus> on pcib27
pcib28: <ACPI PCI-PCI bridge> at device 24.1 on pci0
pci28: <ACPI PCI bus> on pcib28
pcib29: <ACPI PCI-PCI bridge> at device 24.2 on pci0
pci29: <ACPI PCI bus> on pcib29
pcib30: <ACPI PCI-PCI bridge> at device 24.3 on pci0
pci30: <ACPI PCI bus> on pcib30
pcib31: <ACPI PCI-PCI bridge> at device 24.4 on pci0
pci31: <ACPI PCI bus> on pcib31
pcib32: <ACPI PCI-PCI bridge> at device 24.5 on pci0
pci32: <ACPI PCI bus> on pcib32
pcib33: <ACPI PCI-PCI bridge> at device 24.6 on pci0
pci33: <ACPI PCI bus> on pcib33
pcib34: <ACPI PCI-PCI bridge> at device 24.7 on pci0
pci34: <ACPI PCI bus> on pcib34
acpi_acad0: <AC Adapter> on acpi0
atrtc0: <AT realtime clock> port 0x70-0x71 irq 8 on acpi0
atkbdc0: <Keyboard controller (i8042)> port 0x60,0x64 irq 1 on acpi0
atkbd0: <AT Keyboard> irq 1 on atkbdc0
kbd0 at atkbd0
atkbd0: [GIANT-LOCKED]
atkbd0: [ITHREAD]
psm0: <PS/2 Mouse> irq 12 on atkbdc0
psm0: [GIANT-LOCKED]
psm0: [ITHREAD]
psm0: model IntelliMouse, device ID 3
ppc0: <Parallel port> port 0x378-0x37f irq 7 on acpi0
ppc0: Generic chipset (NIBBLE-only) in COMPATIBLE mode
ppc0: [ITHREAD]
ppbus0: <Parallel port bus> on ppc0
plip0: <PLIP network interface> on ppbus0
plip0: [ITHREAD]
lpt0: <Printer> on ppbus0
lpt0: [ITHREAD]
lpt0: Interrupt-driven port
ppi0: <Parallel I/O> on ppbus0
uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0
uart0: [FILTER]
uart1: <16550 or compatible> port 0x2f8-0x2ff irq 3 on acpi0
uart1: [FILTER]
fdc0: <floppy drive controller> port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on =
acpi0
fdc0: [FILTER]
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
cpu0: <ACPI CPU> on acpi0
acpi_throttle0: <ACPI CPU Throttling> on cpu0
orm0: <ISA Option ROMs> at iomem =
0xc0000-0xc7fff,0xca000-0xcafff,0xcb000-0xcbfff,0xcc000-0xccfff,0xcd000-0x=
cdfff,0xdc000-0xdffff,0xe0000-0xe3fff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=3D0x300>
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on =
isa0
ZFS NOTICE: Prefetch is disabled by default if less than 4GB of RAM is =
present;
to enable, add "vfs.zfs.prefetch_disable=3D0" to =
/boot/loader.conf.
ZFS WARNING: Recommended minimum kmem_size is 512MB; expect unstable =
behavior.
Consider tuning vm.kmem_size and vm.kmem_size_max
in /boot/loader.conf.
ZFS filesystem version 3
ZFS storage pool version 14
Timecounter "TSC" frequency 2116630847 Hz quality 800
Timecounters tick every 10.000 msec
Expensive timeout(9) function: 0xffffffff805ed840(0) 0.002412596 s
acd0: DVDR <VMware Virtual IDE CDROM Drive/00000001> at ata1-master =
UDMA33=20
da0 at mpt0 bus 0 scbus0 target 0 lun 0
da0: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da0: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da0: Command Queueing enabled
da0: 8192MB (16777216 512 byte sectors: 255H 63S/T 1044C)
da1 at mpt0 bus 0 scbus0 target 1 lun 0
da1: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da1: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da1: Command Queueing enabled
da1: 4096MB (8388608 512 byte sectors: 255H 63S/T 522C)
da2 at mpt0 bus 0 scbus0 target 2 lun 0
da2: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da2: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da2: Command Queueing enabled
da2: 4096MB (8388608 512 byte sectors: 255H 63S/T 522C)
da3 at mpt0 bus 0 scbus0 target 3 lun 0
da3: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da3: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da3: Command Queueing enabled
da3: 12288MB (25165824 512 byte sectors: 255H 63S/T 1566C)
WARNING: WITNESS option enabled, expect reduced performance.
WARNING: DIAGNOSTIC option enabled, expect reduced performance.
Trying to mount root from ufs:/dev/da0s1a
WARNING: / was not properly dismounted
/: mount pending error: blocks 4 files 1
<118>Setting hostuuid: 564d8c47-d7e5-fea4-92ef-f17760cfe4d2.
<118>Setting hostid: 0x4d345a2c.
lock order reversal:
1st 0xffffff000286d2e8 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:549
2nd 0xffffff000286c0d8 dn->dn_mtx (dn->dn_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:1173
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_block_freed() at dnode_block_freed+0x8e
dbuf_read() at dbuf_read+0x155
dmu_buf_hold_array_by_dnode() at dmu_buf_hold_array_by_dnode+0x12a
dmu_read() at dmu_read+0x80
load_nvlist() at load_nvlist+0x85
spa_load() at spa_load+0x49a
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffffd808, rbp =3D 0x801224140 ---
Expensive timeout(9) function: 0xffffffff80892b50(0xffffff000244fe00) =
0.005049803 s
lock order reversal:
1st 0xffffff0002868e70 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode_sync.c:381
2nd 0xffffff00026e7940 osi->os_lock (osi->os_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:323
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_destroy() at dnode_destroy+0xa6
dnode_buf_pageout() at dnode_buf_pageout+0xb2
dbuf_evict_user() at dbuf_evict_user+0x55
dbuf_clear() at dbuf_clear+0x5e
dnode_evict_dbufs() at dnode_evict_dbufs+0x98
dmu_objset_evict_dbufs() at dmu_objset_evict_dbufs+0x11c
dmu_objset_evict() at dmu_objset_evict+0xbf
dsl_pool_close() at dsl_pool_close+0x52
spa_unload() at spa_unload+0xb2
spa_load() at spa_load+0x4da
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffffd808, rbp =3D 0x801224140 ---
lock order reversal:
1st 0xffffff0002864910 buf->b_lock (buf->b_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/arc.c:2509
2nd 0xffffff00028681a0 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:421
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_read_done() at dbuf_read_done+0x3b
arc_read_done() at arc_read_done+0x1d2
zio_done() at zio_done+0x308
zio_execute() at zio_execute+0xb1
arc_read_nolock() at arc_read_nolock+0x3d0
arc_read() at arc_read+0xaf
dbuf_read() at dbuf_read+0x62b
dmu_buf_hold() at dmu_buf_hold+0xcc
zap_lockdir() at zap_lockdir+0x6e
zap_lookup_norm() at zap_lookup_norm+0x45
zap_lookup() at zap_lookup+0x2e
dsl_prop_get_dd() at dsl_prop_get_dd+0xfc
dsl_dataset_get_ref() at dsl_dataset_get_ref+0x4a7
dsl_dataset_hold() at dsl_dataset_hold+0xf2
dmu_objset_open() at dmu_objset_open+0xcf
zil_check_log_chain() at zil_check_log_chain+0x2a
dmu_objset_find_spa() at dmu_objset_find_spa+0x23f
dmu_objset_find_spa() at dmu_objset_find_spa+0x324
dmu_objset_find_spa() at dmu_objset_find_spa+0x324
dmu_objset_find() at dmu_objset_find+0x2a
spa_check_logs() at spa_check_logs+0x2a
spa_load() at spa_load+0x6ca
spa_load() at spa_load+0x4fe
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffffd808, rbp =3D 0x801224140 ---
lock order reversal:
1st 0xffffff000286d058 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1116
2nd 0xffffff000259ac38 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1120
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_dirty() at dbuf_dirty+0x892
dnode_setdirty() at dnode_setdirty+0x1a9
dbuf_dirty() at dbuf_dirty+0xa53
bplist_vacate() at bplist_vacate+0x4d
spa_sync() at spa_sync+0x297
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff000292d538 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
2nd 0xffffff000250c2f0 spa->spa_sync_bplist.bpl_lock =
(spa->spa_sync_bplist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:235
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
bplist_enqueue_deferred() at bplist_enqueue_deferred+0x47
zio_free() at zio_free+0x105
arc_free() at arc_free+0x11c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x483
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x3eb
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync() at dmu_objset_sync+0x134
dsl_pool_sync() at dsl_pool_sync+0x200
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff0002948c38 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
2nd 0xffffff00028a7000 dn->dn_struct_rwlock (dn->dn_struct_rwlock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1859
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dbuf_check_blkptr() at dbuf_check_blkptr+0x11e
dbuf_sync_list() at dbuf_sync_list+0x34a
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync_dnodes() at dmu_objset_sync_dnodes+0x60
dmu_objset_sync() at dmu_objset_sync+0x162
dsl_pool_sync() at dsl_pool_sync+0x200
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
<118>Entropy harvesting:
<118> interrupts
<118> ethernet
<118> point_to_point
<118> kickstart
<118>.
<118>Starting file system checks:
<118>/dev/da0s1a: FREE BLK COUNT(S) WRONG IN SUPERBLK (SALVAGED)
<118>/dev/da0s1a: SUMMARY INFORMATION BAD (SALVAGED)
<118>/dev/da0s1a: BLK(S) MISSING IN BIT MAPS (SALVAGED)
<118>/dev/da0s1a: 28961 files, 517324 used, 1004493 free (7877 frags, =
124577 blocks, 0.5% fragmentation)
<118>Mounting local file systems:
<118>.
lock order reversal:
1st 0xffffff000263acc8 zfs (zfs) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/gfs.c:437
2nd 0xffffff000256e310 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_root() at zfs_root+0x50
zfsctl_create() at zfsctl_create+0x82
zfs_mount() at zfs_mount+0x7ef
vfs_donmount() at vfs_donmount+0xcde
nmount() at nmount+0x63
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (378, FreeBSD ELF64, nmount), rip =3D 0x800f48f1c, rsp =3D =
0x7fffffffced8, rbp =3D 0x7fffffffcef8 ---
lock order reversal:
1st 0xffffff0002996078 zp->z_name_lock (zp->z_name_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_dir.c:212
2nd 0xffffff000256e330 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_dirent_lock() at zfs_dirent_lock+0x4a0
zfs_dirlook() at zfs_dirlook+0x90
zfs_lookup() at zfs_lookup+0x256
zfs_freebsd_lookup() at zfs_freebsd_lookup+0x8d
VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xaf
vfs_cache_lookup() at vfs_cache_lookup+0xf0
VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xb7
lookup() at lookup+0x3d3
namei() at namei+0x4a9
kern_statat_vnhook() at kern_statat_vnhook+0x8f
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip =3D 0x800fd8acc, rsp =3D =
0x7fffffffcf38, rbp =3D 0x7fffffffd3d0 ---
lock order reversal:
1st 0xffffff000256e210 zfsvfs->z_teardown_inactive_lock =
(zfsvfs->z_teardown_inactive_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_vnops.c:3724
2nd 0xffffff000256e330 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:1027
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zinactive() at zfs_zinactive+0x95
zfs_inactive() at zfs_inactive+0x7e
zfs_freebsd_inactive() at zfs_freebsd_inactive+0x1a
VOP_INACTIVE_APV() at VOP_INACTIVE_APV+0xb5
vinactive() at vinactive+0x90
vputx() at vputx+0x2fc
kern_statat_vnhook() at kern_statat_vnhook+0xfa
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip =3D 0x800fd8acc, rsp =3D =
0x7fffffffcf38, rbp =3D 0x7fffffffd3d0 ---
<118>/etc/rc: WARNING: $hostname is not set -- see rc.conf(5).
<118>Starting Network: lo0 em0 em1 em2 em3.
<118>lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
<118> options=3D3<RXCSUM,TXCSUM>
<118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x6=20
<118> inet6 ::1 prefixlen 128=20
<118> inet 127.0.0.1 netmask 0xff000000=20
<118> nd6 options=3D3<PERFORMNUD,ACCEPT_RTADV>
<118>em0: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:d5
<118> inet 10.0.0.2 netmask 0xffffff00 broadcast 10.0.0.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em1: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:cb
<118> inet 10.0.1.2 netmask 0xffffff00 broadcast 10.0.1.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em2: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:df
<118> inet 10.0.2.2 netmask 0xffffff00 broadcast 10.0.2.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em3: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:e9
<118> inet 172.16.2.118 netmask 0xffff0000 broadcast 172.16.255.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>add net default: gateway 172.16.0.2
Expensive timeout(9) function: 0xffffffff80329580(0xffffff8000272000) =
0.006700013 s
<118>Starting devd.
ipfw2 (+ipv6) initialized, divert loadable, nat loadable, rule-based =
forwarding disabled, default to deny, logging disabled
<118>Flushed all rules.
<118>00100 allow ip from any to any via lo0
<118>00200 deny ip from any to 127.0.0.0/8
<118>00300 deny ip from 127.0.0.0/8 to any
<118>65000 allow ip from any to any
<118>Firewall rules loaded.
<118>Creating and/or trimming log files
<118>.
<118>Starting syslogd.
<118>No core dumps found.
<118>ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib
<118>32-bit compatibility ldconfig path: /usr/lib32
lock order reversal:
1st 0xffffff800a2f62e8 bufwait (bufwait) @ =
/pool/newsrc/src/sys/kern/vfs_bio.c:2559
2nd 0xffffff0002c71400 dirhash (dirhash) @ =
/pool/newsrc/src/sys/ufs/ufs/ufs_dirhash.c:285
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
ufsdirhash_acquire() at ufsdirhash_acquire+0x44
ufsdirhash_add() at ufsdirhash_add+0x19
ufs_direnter() at ufs_direnter+0x88b
ufs_mkdir() at ufs_mkdir+0x623
VOP_MKDIR_APV() at VOP_MKDIR_APV+0xb9
kern_mkdirat() at kern_mkdirat+0x264
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (136, FreeBSD ELF64, mkdir), rip =3D 0x80072bb0c, rsp =3D =
0x7fffffffec88, rbp =3D 0x7fffffffef66 ---
<118>Clearing /tmp (X related).
<118>Updating motd:
<118>.
<118>Configuring syscons:
<118> blanktime
<118>.
<118>Starting sshd.
<118>Starting cron.
<118>Starting background file system checks in 60 seconds.
<118>
<118>Wed Mar 10 17:29:09 CET 2010
<118>Mar 10 17:29:21 su: BAD SU borjam to root on /dev/pts/0
<118>Mar 10 17:29:24 su: BAD SU borjam to root on /dev/pts/0
<118>Mar 10 17:29:26 su: borjam to root on /dev/pts/0
<118>Mar 10 17:30:36 su: borjam to root on /dev/pts/1
lock order reversal:
1st 0xffffff00027e2438 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:189
2nd 0xffffff000286bb88 dn->dn_struct_rwlock (dn->dn_struct_rwlock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:130
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dnode_verify() at dnode_verify+0x70
dnode_hold_impl() at dnode_hold_impl+0x73
dmu_bonus_hold() at dmu_bonus_hold+0x31
bplist_hold() at bplist_hold+0x48
bplist_enqueue() at bplist_enqueue+0x4c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x3eb
dbuf_sync_list() at dbuf_sync_list+0x17f
Fatal double fault
rip =3D 0xffffffff805d322d
rsp =3D 0xffffff8012414fe0
rbp =3D 0xffffff8012415090
cpuid =3D 0; apic id =3D 00
panic: double fault
cpuid =3D 0
KDB: enter: panic
shared sx zfsvfs->z_teardown_inactive_lock =
(zfsvfs->z_teardown_inactive_lock) r =3D 0 (0xffffff000256d210) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_vnops.c:3724
exclusive lockmgr zfs (zfs) r =3D 0 (0xffffff00055b3a58) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive lockmgr zfs (zfs) r =3D 0 (0xffffff0005618308) locked @ =
/pool/newsrc/src/sys/kern/vfs_syscalls.c:3174
shared lockmgr zfs (zfs) r =3D 0 (0xffffff00057bda58) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive sx zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) r =3D 0 =
(0xffffff000256d310) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:866
shared sx zp->z_name_lock (zp->z_name_lock) r =3D 0 (0xffffff00057c7c38) =
locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_dir.c:212
exclusive lockmgr zfs (zfs) r =3D 0 (0xffffff00057ac578) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive sx so_rcv_sx (so_rcv_sx) r =3D 0 (0xffffff0002f63648) locked @ =
/pool/newsrc/src/sys/kern/uipc_sockbuf.c:148
exclusive sx so_rcv_sx (so_rcv_sx) r =3D 0 (0xffffff0002cff3a0) locked @ =
/pool/newsrc/src/sys/kern/uipc_sockbuf.c:148
exclusive sx ds->ds_deadlist.bpl_lock (ds->ds_deadlist.bpl_lock) r =3D 0 =
(0xffffff00027e2438) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:189
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r =3D 0 =
(0xffffff0002948238) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r =3D 0 =
(0xffffff0002963538) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r =3D 0 =
(0xffffff0002949438) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r =3D 0 =
(0xffffff0002948b38) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r =3D 0 =
(0xffffff0002966238) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
exclusive sx dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) r =3D 0 =
(0xffffff0002963438) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
0xffffff00055b39c0: tag zfs, type VDIR
usecount 0, writecount 0, refcount 1 mountedhere 0
flags (VI_DOINGINACT)
lock type zfs: EXCL by thread 0xffffff000528b3a0 (pid 1425)
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff806255cf at kern_statat_vnhook+0x8f
#11 0xffffffff80625775 at kern_statat+0x15
#12 0xffffffff8062583a at lstat+0x2a
#13 0xffffffff80876b48 at syscall+0x118
#14 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff0005618270: tag zfs, type VDIR
usecount 1, writecount 0, refcount 1 mountedhere 0
flags ()
lock type zfs: EXCL by thread 0xffffff000528c000 (pid 1422)
#0 0xffffffff80578f77 at __lockmgr_args+0x777
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806299ba at setutimes+0x7a
#5 0xffffffff80629cb2 at kern_lutimes+0xf2
#6 0xffffffff80876b48 at syscall+0x118
#7 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff00057bd9c0: tag zfs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type zfs: SHARED (count 1)
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff806255cf at kern_statat_vnhook+0x8f
#11 0xffffffff80625775 at kern_statat+0x15
#12 0xffffffff8062583a at lstat+0x2a
#13 0xffffffff80876b48 at syscall+0x118
#14 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff00057ac4e0: tag zfs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type zfs: EXCL by thread 0xffffff000262a3a0 (pid 1416)
with shared waiters pending
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff80624df3 at kern_mkdirat+0xc3
#11 0xffffffff80876b48 at syscall+0x118
#12 0xffffffff8085cf91 at Xfast_syscall+0xe1
Uptime: 2m16s
Textdump complete.
Automatic reboot in 15 seconds - press a key on the console to abort
--> Press a key on the console to reboot,
--> or switch off the system now.
Rebooting...
Copyright (c) 1992-2010 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights =
reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 8.0-STABLE #3: Wed Mar 10 14:41:54 CET 2010
=
root@pruebazfs1.sarenet.es:/pool/newsrc/obj/pool/newsrc/src/sys/DEBUG =
amd64
WARNING: WITNESS option enabled, expect reduced performance.
WARNING: DIAGNOSTIC option enabled, expect reduced performance.
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Intel(R) Core(TM)2 Duo CPU T8100 @ 2.10GHz (2116.63-MHz =
K8-class CPU)
Origin =3D "GenuineIntel" Id =3D 0x10676 Stepping =3D 6
=
Features=3D0xfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,=
MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS>
Features2=3D0x80082201<SSE3,SSSE3,CX16,SSE4.1,<b31>>
AMD Features=3D0x20100800<SYSCALL,NX,LM>
AMD Features2=3D0x1<LAHF>
TSC: P-state invariant
real memory =3D 536870912 (512 MB)
avail memory =3D 495988736 (473 MB)
ACPI APIC Table: <PTLTD APIC >
MADT: Forcing active-low polarity and level trigger for SCI
ioapic0 <Version 1.1> irqs 0-23 on motherboard
kbd1 at kbdmux0
acpi0: <INTEL 440BX> on motherboard
acpi0: [ITHREAD]
acpi0: Power Button (fixed)
Timecounter "ACPI-safe" frequency 3579545 Hz quality 850
acpi_timer0: <24-bit timer at 3.579545MHz> port 0x1008-0x100b on acpi0
pcib0: <ACPI Host-PCI bridge> port 0xcf8-0xcff on acpi0
pci0: <ACPI PCI bus> on pcib0
pcib1: <ACPI PCI-PCI bridge> at device 1.0 on pci0
pci1: <ACPI PCI bus> on pcib1
isab0: <PCI-ISA bridge> at device 7.0 on pci0
isa0: <ISA bus> on isab0
atapci0: <Intel PIIX4 UDMA33 controller> port =
0x1f0-0x1f7,0x3f6,0x170-0x177,0x376,0x10c0-0x10cf at device 7.1 on pci0
ata0: <ATA channel 0> on atapci0
ata0: [ITHREAD]
ata1: <ATA channel 1> on atapci0
ata1: [ITHREAD]
pci0: <bridge> at device 7.3 (no driver attached)
pci0: <base peripheral> at device 7.7 (no driver attached)
vgapci0: <VGA-compatible display> port 0x10d0-0x10df mem =
0xd0000000-0xd7ffffff,0xd8000000-0xd87fffff irq 16 at device 15.0 on =
pci0
mpt0: <LSILogic 1030 Ultra4 Adapter> port 0x1400-0x14ff mem =
0xd8820000-0xd883ffff,0xd8800000-0xd881ffff irq 17 at device 16.0 on =
pci0
mpt0: [ITHREAD]
mpt0: MPI Version=3D1.2.0.0
pcib2: <ACPI PCI-PCI bridge> at device 17.0 on pci0
pci2: <ACPI PCI bus> on pcib2
em0: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x2000-0x203f =
mem 0xd8940000-0xd895ffff,0xd8900000-0xd890ffff irq 18 at device 0.0 on =
pci2
em0: Memory Access and/or Bus Master bits were not set!
em0: [FILTER]
em0: Ethernet address: 00:0c:29:11:4f:d5
em1: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x2040-0x207f =
mem 0xd8960000-0xd897ffff,0xd8910000-0xd891ffff irq 19 at device 1.0 on =
pci2
em1: Memory Access and/or Bus Master bits were not set!
em1: [FILTER]
em1: Ethernet address: 00:0c:29:11:4f:cb
pci2: <multimedia, audio> at device 2.0 (no driver attached)
em2: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x20c0-0x20ff =
mem 0xd8980000-0xd899ffff,0xd8920000-0xd892ffff irq 17 at device 3.0 on =
pci2
em2: Memory Access and/or Bus Master bits were not set!
em2: [FILTER]
em2: Ethernet address: 00:0c:29:11:4f:df
em3: <Intel(R) PRO/1000 Network Connection 6.9.14> port 0x2400-0x243f =
mem 0xd89a0000-0xd89bffff,0xd8930000-0xd893ffff irq 19 at device 5.0 on =
pci2
em3: Memory Access and/or Bus Master bits were not set!
em3: [FILTER]
em3: Ethernet address: 00:0c:29:11:4f:e9
pcib3: <ACPI PCI-PCI bridge> at device 21.0 on pci0
pci3: <ACPI PCI bus> on pcib3
pcib4: <ACPI PCI-PCI bridge> at device 21.1 on pci0
pci4: <ACPI PCI bus> on pcib4
pcib5: <ACPI PCI-PCI bridge> at device 21.2 on pci0
pci5: <ACPI PCI bus> on pcib5
pcib6: <ACPI PCI-PCI bridge> at device 21.3 on pci0
pci6: <ACPI PCI bus> on pcib6
pcib7: <ACPI PCI-PCI bridge> at device 21.4 on pci0
pci7: <ACPI PCI bus> on pcib7
pcib8: <ACPI PCI-PCI bridge> at device 21.5 on pci0
pci8: <ACPI PCI bus> on pcib8
pcib9: <ACPI PCI-PCI bridge> at device 21.6 on pci0
pci9: <ACPI PCI bus> on pcib9
pcib10: <ACPI PCI-PCI bridge> at device 21.7 on pci0
pci10: <ACPI PCI bus> on pcib10
pcib11: <ACPI PCI-PCI bridge> at device 22.0 on pci0
pci11: <ACPI PCI bus> on pcib11
pcib12: <ACPI PCI-PCI bridge> at device 22.1 on pci0
pci12: <ACPI PCI bus> on pcib12
pcib13: <ACPI PCI-PCI bridge> at device 22.2 on pci0
pci13: <ACPI PCI bus> on pcib13
pcib14: <ACPI PCI-PCI bridge> at device 22.3 on pci0
pci14: <ACPI PCI bus> on pcib14
pcib15: <ACPI PCI-PCI bridge> at device 22.4 on pci0
pci15: <ACPI PCI bus> on pcib15
pcib16: <ACPI PCI-PCI bridge> at device 22.5 on pci0
pci16: <ACPI PCI bus> on pcib16
pcib17: <ACPI PCI-PCI bridge> at device 22.6 on pci0
pci17: <ACPI PCI bus> on pcib17
pcib18: <ACPI PCI-PCI bridge> at device 22.7 on pci0
pci18: <ACPI PCI bus> on pcib18
pcib19: <ACPI PCI-PCI bridge> at device 23.0 on pci0
pci19: <ACPI PCI bus> on pcib19
pcib20: <ACPI PCI-PCI bridge> at device 23.1 on pci0
pci20: <ACPI PCI bus> on pcib20
pcib21: <ACPI PCI-PCI bridge> at device 23.2 on pci0
pci21: <ACPI PCI bus> on pcib21
pcib22: <ACPI PCI-PCI bridge> at device 23.3 on pci0
pci22: <ACPI PCI bus> on pcib22
pcib23: <ACPI PCI-PCI bridge> at device 23.4 on pci0
pci23: <ACPI PCI bus> on pcib23
pcib24: <ACPI PCI-PCI bridge> at device 23.5 on pci0
pci24: <ACPI PCI bus> on pcib24
pcib25: <ACPI PCI-PCI bridge> at device 23.6 on pci0
pci25: <ACPI PCI bus> on pcib25
pcib26: <ACPI PCI-PCI bridge> at device 23.7 on pci0
pci26: <ACPI PCI bus> on pcib26
pcib27: <ACPI PCI-PCI bridge> at device 24.0 on pci0
pci27: <ACPI PCI bus> on pcib27
pcib28: <ACPI PCI-PCI bridge> at device 24.1 on pci0
pci28: <ACPI PCI bus> on pcib28
pcib29: <ACPI PCI-PCI bridge> at device 24.2 on pci0
pci29: <ACPI PCI bus> on pcib29
pcib30: <ACPI PCI-PCI bridge> at device 24.3 on pci0
pci30: <ACPI PCI bus> on pcib30
pcib31: <ACPI PCI-PCI bridge> at device 24.4 on pci0
pci31: <ACPI PCI bus> on pcib31
pcib32: <ACPI PCI-PCI bridge> at device 24.5 on pci0
pci32: <ACPI PCI bus> on pcib32
pcib33: <ACPI PCI-PCI bridge> at device 24.6 on pci0
pci33: <ACPI PCI bus> on pcib33
pcib34: <ACPI PCI-PCI bridge> at device 24.7 on pci0
pci34: <ACPI PCI bus> on pcib34
acpi_acad0: <AC Adapter> on acpi0
atrtc0: <AT realtime clock> port 0x70-0x71 irq 8 on acpi0
atkbdc0: <Keyboard controller (i8042)> port 0x60,0x64 irq 1 on acpi0
atkbd0: <AT Keyboard> irq 1 on atkbdc0
kbd0 at atkbd0
atkbd0: [GIANT-LOCKED]
atkbd0: [ITHREAD]
psm0: <PS/2 Mouse> irq 12 on atkbdc0
psm0: [GIANT-LOCKED]
psm0: [ITHREAD]
psm0: model IntelliMouse, device ID 3
ppc0: <Parallel port> port 0x378-0x37f irq 7 on acpi0
ppc0: Generic chipset (NIBBLE-only) in COMPATIBLE mode
ppc0: [ITHREAD]
ppbus0: <Parallel port bus> on ppc0
plip0: <PLIP network interface> on ppbus0
plip0: [ITHREAD]
lpt0: <Printer> on ppbus0
lpt0: [ITHREAD]
lpt0: Interrupt-driven port
ppi0: <Parallel I/O> on ppbus0
uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0
uart0: [FILTER]
uart1: <16550 or compatible> port 0x2f8-0x2ff irq 3 on acpi0
uart1: [FILTER]
fdc0: <floppy drive controller> port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on =
acpi0
fdc0: [FILTER]
fd0: <1440-KB 3.5" drive> on fdc0 drive 0
cpu0: <ACPI CPU> on acpi0
acpi_throttle0: <ACPI CPU Throttling> on cpu0
orm0: <ISA Option ROMs> at iomem =
0xc0000-0xc7fff,0xca000-0xcafff,0xcb000-0xcbfff,0xcc000-0xccfff,0xcd000-0x=
cdfff,0xdc000-0xdffff,0xe0000-0xe3fff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=3D0x300>
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on =
isa0
ZFS NOTICE: Prefetch is disabled by default if less than 4GB of RAM is =
present;
to enable, add "vfs.zfs.prefetch_disable=3D0" to =
/boot/loader.conf.
ZFS WARNING: Recommended minimum kmem_size is 512MB; expect unstable =
behavior.
Consider tuning vm.kmem_size and vm.kmem_size_max
in /boot/loader.conf.
ZFS filesystem version 3
ZFS storage pool version 14
Timecounter "TSC" frequency 2116629050 Hz quality 800
Timecounters tick every 10.000 msec
acd0: DVDR <VMware Virtual IDE CDROM Drive/00000001> at ata1-master =
UDMA33=20
Expensive timeout(9) function: 0xffffffff805ed840(0) 0.002413714 s
Expensive timeout(9) function: 0xffffffff80885bf0(0xffffffff80e68900) =
0.003080000 s
da0 at mpt0 bus 0 scbus0 target 0 lun 0
da0: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da0: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da0: Command Queueing enabled
da0: 8192MB (16777216 512 byte sectors: 255H 63S/T 1044C)
da1 at mpt0 bus 0 scbus0 target 1 lun 0
da1: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da1: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da1: Command Queueing enabled
da1: 4096MB (8388608 512 byte sectors: 255H 63S/T 522C)
da2 at mpt0 bus 0 scbus0 target 2 lun 0
da2: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da2: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da2: Command Queueing enabled
da2: 4096MB (8388608 512 byte sectors: 255H 63S/T 522C)
da3 at mpt0 bus 0 scbus0 target 3 lun 0
da3: <VMware, VMware Virtual S 1.0> Fixed Direct Access SCSI-2 device=20
da3: 320.000MB/s transfers (160.000MHz, offset 127, 16bit)
da3: Command Queueing enabled
da3: 12288MB (25165824 512 byte sectors: 255H 63S/T 1566C)
WARNING: WITNESS option enabled, expect reduced performance.
WARNING: DIAGNOSTIC option enabled, expect reduced performance.
Trying to mount root from ufs:/dev/da0s1a
WARNING: / was not properly dismounted
<118>Setting hostuuid: 564d8c47-d7e5-fea4-92ef-f17760cfe4d2.
<118>Setting hostid: 0x4d345a2c.
lock order reversal:
1st 0xffffff00028732e8 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:549
2nd 0xffffff00028720d8 dn->dn_mtx (dn->dn_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:1173
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_block_freed() at dnode_block_freed+0x8e
dbuf_read() at dbuf_read+0x155
dmu_buf_hold_array_by_dnode() at dmu_buf_hold_array_by_dnode+0x12a
dmu_read() at dmu_read+0x80
load_nvlist() at load_nvlist+0x85
spa_load() at spa_load+0x49a
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
Expensive timeout(9) function: 0xffffffff80892b50(0xffffff000244fe00) =
0.003086146 s
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffffd808, rbp =3D 0x801224140 ---
lock order reversal:
1st 0xffffff0002868e70 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode_sync.c:381
2nd 0xffffff0002767940 osi->os_lock (osi->os_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:323
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_destroy() at dnode_destroy+0xa6
dnode_buf_pageout() at dnode_buf_pageout+0xb2
dbuf_evict_user() at dbuf_evict_user+0x55
dbuf_clear() at dbuf_clear+0x5e
dnode_evict_dbufs() at dnode_evict_dbufs+0x98
dmu_objset_evict_dbufs() at dmu_objset_evict_dbufs+0x11c
dmu_objset_evict() at dmu_objset_evict+0xbf
dsl_pool_close() at dsl_pool_close+0x52
spa_unload() at spa_unload+0xb2
spa_load() at spa_load+0x4da
spa_open_common() at spa_open_common+0x12d
spa_get_stats() at spa_get_stats+0x42
zfs_ioc_pool_stats() at zfs_ioc_pool_stats+0x2c
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffffd808, rbp =3D 0x801224140 ---
lock order reversal:
1st 0xffffff0002873058 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1116
2nd 0xffffff000259ac38 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1120
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_sExpensive timeout(9) function: =
0xffffffff805ed8c0(0) 0.003133917 s
elf_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_dirty() at dbuf_dirty+0x892
dnode_setdirty() at dnode_setdirty+0x1a9
dbuf_dirty() at dbuf_dirty+0xa53
bplist_vacate() at bplist_vacate+0x4d
spa_sync() at spa_sync+0x297
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff0002926838 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
2nd 0xffffff000250c2f0 spa->spa_sync_bplist.bpl_lock =
(spa->spa_sync_bplist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:235
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
bplist_enqueue_deferred() at bplist_enqueue_deferred+0x47
zio_free() at zio_free+0x105
arc_free() at arc_free+0x11c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x483
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x3eb
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync() at dmu_objset_sync+0x134
dsl_pool_sync() at dsl_pool_sync+0x200
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at Expensive timeout(9) function: =
0xffffffff80885bf0(0xffffffff80e68900) 0.003364952 s
fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
<118>Entropy harvesting:
<118> interrupts
<118> ethernet
<118> point_to_point
<118> kickstart
<118>.
<118>Starting file system checks:
<118>/dev/da0s1a: INCORRECT BLOCK COUNT I=3D259208 (16 should be 12) =
(CORRECTED)
<118>/dev/da0s1a: LINK COUNT DIR I=3D235521 OWNER=3Droot MODE=3D41777
<118>/dev/da0s1a: SIZE=3D3584 MTIME=3DMar 10 17:30 2010 COUNT 120 =
SHOULD BE 119 (ADJUSTED)
<118>/dev/da0s1a: SUMMARY INFORMATION BAD (SALVAGED)
<118>/dev/da0s1a: BLK(S) MISSING IN BIT MAPS (SALVAGED)
<118>/dev/da0s1a: 28967 files, 517344 used, 1004473 free (7889 frags, =
124573 blocks, 0.5% fragmentation)
Expensive timeout(9) function: 0xffffffff80475420(0xffffffff80e77f60) =
0.004632152 s
<118>Mounting local file systems:
<118>.
lock order reversal:
1st 0xffffff0002620cc8 zfs (zfs) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/gfs.c:437
2nd 0xffffff000256f310 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_root() at zfs_root+0x50
zfsctl_create() at zfsctl_create+0x82
zfs_mount() at zfs_mount+0x7ef
vfs_donmount() at vfs_donmount+0xcde
nmount() at nmount+0x63
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (378, FreeBSD ELF64, nmount), rip =3D 0x800f48f1c, rsp =3D =
0x7fffffffced8, rbp =3D 0x7fffffffcef8 ---
lock order reversal:
1st 0xffffff0002655078 zp->z_name_lock (zp->z_name_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_dir.c:212
2nd 0xffffff000256f330 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_dirent_lock() at zfs_dirent_lock+0x4a0
zfs_dirlook() at zfs_dirlook+0x90
zfs_lookup() at zfs_lookup+0x256
zfs_freebsd_lookup() at zfs_freebsd_lookup+0x8d
VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xaf
vfs_cache_lookup() at vfs_cache_lookup+0xf0
VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xb7
lookup() at lookup+0x3d3
namei() at namei+0x4a9
kern_statat_vnhook() at kern_statat_vnhook+0x8f
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip =3D 0x800fd8acc, rsp =3D =
0x7fffffffcf38, rbp =3D 0x7fffffffd3d0 ---
lock order reversal:
1st 0xffffff000256f210 zfsvfs->z_teardown_inactive_lock =
(zfsvfs->z_teardown_inactive_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_vnops.c:3724
2nd 0xffffff000256f330 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:1027
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zinactive() at zfs_zinactive+0x95
zfs_inactive() at zfs_inactive+0x7e
zfs_freebsd_inactive() at zfs_freebsd_inactive+0x1a
VOP_INACTIVE_APV() at VOP_INACTIVE_APV+0xb5
vinactive() at vinactive+0x90
vputx() at vputx+0x2fc
kern_statat_vnhook() at kern_statat_vnhook+0xfa
kern_statat() at kern_statat+0x15
lstat() at lstat+0x2a
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (190, FreeBSD ELF64, lstat), rip =3D 0x800fd8acc, rsp =3D =
0x7fffffffcf38, rbp =3D 0x7fffffffd3d0 ---
<118>/etc/rc: WARNING: $hostname is not set -- see rc.conf(5).
<118>Starting Network: lo0 em0 em1 em2 em3.
<118>lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
<118> options=3D3<RXCSUM,TXCSUM>
<118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x6=20
<118> inet6 ::1 prefixlen 128=20
<118> inet 127.0.0.1 netmask 0xff000000=20
<118> nd6 options=3D3<PERFORMNUD,ACCEPT_RTADV>
<118>em0: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:d5
<118> inet 10.0.0.2 netmask 0xffffff00 broadcast 10.0.0.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em1: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:cb
<118> inet 10.0.1.2 netmask 0xffffff00 broadcast 10.0.1.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em2: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:df
<118> inet 10.0.2.2 netmask 0xffffff00 broadcast 10.0.2.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>em3: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 =
mtu 1500
<118> options=3D9b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM>
<118> ether 00:0c:29:11:4f:e9
<118> inet 172.16.2.118 netmask 0xffff0000 broadcast 172.16.255.255
<118> media: Ethernet autoselect (1000baseT <full-duplex>)
<118> status: active
<118>add net default: gateway 172.16.0.2
<118>Starting devd.
Expensive timeout(9) function: 0xffffffff80329580(0xffffff8000284000) =
0.010275607 s
ipfw2 (+ipv6) initialized, divert loadable, nat loadable, rule-based =
forwarding disabled, default to deny, logging disabled
<118>Flushed all rules.
<118>00100 allow ip from any to any via lo0
<118>00200 deny ip from any to 127.0.0.0/8
<118>00300 deny ip from 127.0.0.0/8 to any
<118>65000 allow ip from any to any
<118>Firewall rules loaded.
<118>Creating and/or trimming log files
<118>.
<118>Starting syslogd.
<118>savecore: reboot after panic: double fault
<118>Mar 10 17:33:17 savecore: reboot after panic: double fault
<118>savecore: writing core to textdump.tar.1
<118>/var/crash/vmcore.1 not found
<118>ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib
<118>32-bit compatibility ldconfig path: /usr/lib32
lock order reversal:
1st 0xffffff800a2f8338 bufwait (bufwait) @ =
/pool/newsrc/src/sys/kern/vfs_bio.c:2559
2nd 0xffffff0002ca5c00 dirhash (dirhash) @ =
/pool/newsrc/src/sys/ufs/ufs/ufs_dirhash.c:285
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
ufsdirhash_acquire() at ufsdirhash_acquire+0x44
ufsdirhash_add() at ufsdirhash_add+0x19
ufs_direnter() at ufs_direnter+0x88b
ufs_mkdir() at ufs_mkdir+0x623
VOP_MKDIR_APV() at VOP_MKDIR_APV+0xb9
kern_mkdirat() at kern_mkdirat+0x264
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (136, FreeBSD ELF64, mkdir), rip =3D 0x80072bb0c, rsp =3D =
0x7fffffffec88, rbp =3D 0x7fffffffef66 ---
<118>Clearing /tmp (X related).
<118>Updating motd:
<118>.
<118>Configuring syscons:
<118> blanktime
<118>.
<118>Starting sshd.
lock order reversal:
1st 0xffffff0002864b08 buf->b_lock (buf->b_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/arc.c:2509
2nd 0xffffff00025e9430 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:421
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_read_done() at dbuf_read_done+0x3b
arc_read_nolock() at arc_read_nolock+0x526
arc_read() at arc_read+0xaf
dbuf_read() at dbuf_read+0x62b
dmu_buf_hold() at dmu_buf_hold+0xcc
bplist_cache() at bplist_cache+0x4c
bplist_iterate() at bplist_iterate+0xb3
spa_sync() at spa_sync+0x262
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
<118>Starting cron.
<118>Starting background file system checks in 60 seconds.
<118>
<118>Wed Mar 10 17:33:21 CET 2010
<118>Mar 10 17:33:30 su: borjam to root on /dev/pts/0
lock order reversal:
1st 0xffffff0002c6d238 dr->dt.di.dr_mtx (dr->dt.di.dr_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1905
2nd 0xffffff0002866000 dn->dn_struct_rwlock (dn->dn_struct_rwlock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:543
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dbuf_read() at dbuf_read+0x2ad
dbuf_will_dirty() at dbuf_will_dirty+0x53
dsl_dataset_block_kill() at dsl_dataset_block_kill+0xe9
dbuf_write() at dbuf_write+0x24c
dbuf_sync_list() at dbuf_sync_list+0x159
dbuf_sync_list() at dbuf_sync_list+0x17f
dnode_sync() at dnode_sync+0xc12
dmu_objset_sync() at dmu_objset_sync+0x134
dsl_pool_sync() at dsl_pool_sync+0x88
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff000248d038 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:189
2nd 0xffffff0002871b88 dn->dn_struct_rwlock (dn->dn_struct_rwlock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:130
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_slock() at _sx_slock+0x55
dnode_verify() at dnode_verify+0x70
dnode_hold_impl() at dnode_hold_impl+0x73
dmu_bonus_hold() at dmu_bonus_hold+0x31
bplist_hold() at bplist_hold+0x48
bplist_enqueue() at bplist_enqueue+0x4c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dmu_objset_sync() at dmu_objset_sync+0x1fe
dsl_pool_sync() at dsl_pool_sync+0x88
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff000248d038 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:189
2nd 0xffffff0002f340d8 dn->dn_mtx (dn->dn_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:606
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_hold_impl() at dnode_hold_impl+0x184
dmu_bonus_hold() at dmu_bonus_hold+0x31
bplist_hold() at bplist_hold+0x48
bplist_enqueue() at bplist_enqueue+0x4c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dmu_objset_sync() at dmu_objset_sync+0x1fe
dsl_pool_sync() at dsl_pool_sync+0x88
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff000248d038 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:189
2nd 0xffffff00028686c0 db->db_mtx (db->db_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1724
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_rele() at dbuf_rele+0x2d
dnode_hold_impl() at dnode_hold_impl+0x20f
dmu_bonus_hold() at dmu_bonus_hold+0x31
bplist_hold() at bplist_hold+0x48
bplist_enqueue() at bplist_enqueue+0x4c
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dmu_objset_sync() at dmu_objset_sync+0x1fe
dsl_pool_sync() at dsl_pool_sync+0x88
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff000248d038 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:189
2nd 0xffffff00027d9540 osi->os_lock (osi->os_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dnode.c:687
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dnode_setdirty() at dnode_setdirty+0xbc
dbuf_dirty() at dbuf_dirty+0x516
bplist_enqueue() at bplist_enqueue+0xbd
dsl_dataset_block_kill() at dsl_dataset_block_kill+0x119
dmu_objset_sync() at dmu_objset_sync+0x1fe
dsl_pool_sync() at dsl_pool_sync+0x88
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff0002893210 zfsvfs->z_teardown_inactive_lock =
(zfsvfs->z_teardown_inactive_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_vfsops.c:917
2nd 0xffffff000248d0f8 ds->ds_rwlock (ds->ds_rwlock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dsl_dataset.c:2864
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dsl_dataset_clone_swap() at dsl_dataset_clone_swap+0x5a
dmu_recv_end() at dmu_recv_end+0x94
zfs_ioc_recv() at zfs_ioc_recv+0x29d
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffff8e98, rbp =3D 0x7fffffff9bd0 ---
lock order reversal:
1st 0xffffff000248d038 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:152
2nd 0xffffff0002f34330 dn->dn_dbufs_mtx (dn->dn_dbufs_mtx) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:1518
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_destroy() at dbuf_destroy+0x58
bplist_cache() at bplist_cache+0x2e
bplist_iterate() at bplist_iterate+0xb3
bplist_space_birthrange() at bplist_space_birthrange+0x60
dsl_dataset_clone_swap_sync() at dsl_dataset_clone_swap_sync+0xee
dsl_sync_task_group_sync() at dsl_sync_task_group_sync+0x173
dsl_pool_sync() at dsl_pool_sync+0x122
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff000248d038 ds->ds_deadlist.bpl_lock =
(ds->ds_deadlist.bpl_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/bplist.c:152
2nd 0xffffffff81150db0 h->hash_mutexes[i] (h->hash_mutexes[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dbuf.c:191
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
dbuf_destroy() at dbuf_destroy+0x111
bplist_cache() at bplist_cache+0x2e
bplist_iterate() at bplist_iterate+0xb3
bplist_space_birthrange() at bplist_space_birthrange+0x60
dsl_dataset_clone_swap_sync() at dsl_dataset_clone_swap_sync+0xee
dsl_sync_task_group_sync() at dsl_sync_task_group_sync+0x173
dsl_pool_sync() at dsl_pool_sync+0x122
spa_sync() at spa_sync+0x35e
txg_sync_thread() at txg_sync_thread+0x2d7
fork_exit() at fork_exit+0x12a
fork_trampoline() at fork_trampoline+0xe
--- trap 0, rip =3D 0, rsp =3D 0xffffff8012417d30, rbp =3D 0 ---
lock order reversal:
1st 0xffffff0002893250 zfsvfs->z_znodes_lock (zfsvfs->z_znodes_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_vfsops.c:1317
2nd 0xffffff0002893310 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:966
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_rezget() at zfs_rezget+0x4a
zfs_resume_fs() at zfs_resume_fs+0x158
zfs_ioc_recv() at zfs_ioc_recv+0x2b4
zfsdev_ioctl() at zfsdev_ioctl+0x8d
devfs_ioctl_f() at devfs_ioctl_f+0x76
kern_ioctl() at kern_ioctl+0xc5
ioctl() at ioctl+0xfd
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (54, FreeBSD ELF64, ioctl), rip =3D 0x800fe7d1c, rsp =3D =
0x7fffffff8e98, rbp =3D 0x7fffffff9bd0 ---
lock order reversal:
1st 0xffffff0004e911d0 zp->z_parent_lock (zp->z_parent_lock) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_dir.c:379
2nd 0xffffff0002893350 zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:866
KDB: stack backtrace:
db_trace_self_wrapper() at db_trace_self_wrapper+0x2a
_witness_debugger() at _witness_debugger+0x2e
witness_checkorder() at witness_checkorder+0x81e
_sx_xlock() at _sx_xlock+0x55
zfs_zget() at zfs_zget+0x23c
zfs_dirlook() at zfs_dirlook+0x1fc
zfs_lookup() at zfs_lookup+0x256
zfs_freebsd_lookup() at zfs_freebsd_lookup+0x8d
VOP_CACHEDLOOKUP_APV() at VOP_CACHEDLOOKUP_APV+0xaf
vfs_cache_lookup() at vfs_cache_lookup+0xf0
VOP_LOOKUP_APV() at VOP_LOOKUP_APV+0xb7
lookup() at lookup+0x3d3
namei() at namei+0x4a9
kern_chdir() at kern_chdir+0x78
syscall() at syscall+0x118
Xfast_syscall() at Xfast_syscall+0xe1
--- syscall (12, FreeBSD ELF64, chdir), rip =3D 0x800da74dc, rsp =3D =
0x7fffffffe898, rbp =3D 0x801046f10 ---
KDB: enter: sysctl debug.kdb.enter
exclusive sleep mutex Giant (Giant) r =3D 0 (0xffffffff80c4c540) locked =
@ /pool/newsrc/src/sys/kern/kern_sysctl.c:1417
shared sx sysctl lock (sysctl lock) r =3D 0 (0xffffffff80c4cec0) locked =
@ /pool/newsrc/src/sys/kern/kern_sysctl.c:1521
exclusive sx zfsvfs->z_teardown_inactive_lock =
(zfsvfs->z_teardown_inactive_lock) r =3D 0 (0xffffff0002893210) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_vfsops.c:917
shared sx ds->ds_rwlock (ds->ds_rwlock) r =3D 0 (0xffffff00190f50f8) =
locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/dsl_dataset.c:519
exclusive sx zfsvfs->z_online_recv_lock (zfsvfs->z_online_recv_lock) r =3D=
0 (0xffffff0002893288) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_ioctl.c:2492
exclusive lockmgr zfs (zfs) r =3D 0 (0xffffff0007d43308) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:888
exclusive sx zfsvfs->z_hold_mtx[i] (zfsvfs->z_hold_mtx[i]) r =3D 0 =
(0xffffff00028963f0) locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_znode.c:768
shared sx zp->z_name_lock (zp->z_name_lock) r =3D 0 (0xffffff0007cf31f0) =
locked @ =
/pool/newsrc/src/sys/modules/zfs/../../cddl/contrib/opensolaris/uts/common=
/fs/zfs/zfs_dir.c:212
exclusive lockmgr zfs (zfs) r =3D 0 (0xffffff0007d11098) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:2091
exclusive sx so_rcv_sx (so_rcv_sx) r =3D 0 (0xffffff0002c763a0) locked @ =
/pool/newsrc/src/sys/kern/uipc_sockbuf.c:148
exclusive lockmgr syncer (syncer) r =3D 0 (0xffffff0002947cc8) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:1701
exclusive lockmgr zfs (zfs) r =3D 0 (0xffffff0007d61cc8) locked @ =
/pool/newsrc/src/sys/kern/vfs_subr.c:888
0xffffff0007d11000: tag zfs, type VDIR
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type zfs: EXCL by thread 0xffffff000262dae0 (pid 1699)
#0 0xffffffff80578cef at __lockmgr_args+0x4ef
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806210bb at vget+0x7b
#5 0xffffffff8060e570 at cache_lookup+0x4e0
#6 0xffffffff8060e890 at vfs_cache_lookup+0xc0
#7 0xffffffff808cbdb7 at VOP_LOOKUP_APV+0xb7
#8 0xffffffff80615513 at lookup+0x3d3
#9 0xffffffff806164e9 at namei+0x4a9
#10 0xffffffff8062d1a2 at vn_open_cred+0xc2
#11 0xffffffff8062b569 at kern_openat+0x179
#12 0xffffffff80876b48 at syscall+0x118
#13 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff0002947c30: tag syncer, type VNON
usecount 1, writecount 0, refcount 2 mountedhere 0
flags ()
lock type syncer: EXCL by thread 0xffffff000250d3a0 (pid 18)
#0 0xffffffff80578f77 at __lockmgr_args+0x777
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff8062d957 at _vn_lock+0x57
#4 0xffffffff806229a0 at sync_vnode+0x130
#5 0xffffffff80622c2c at sched_sync+0x1cc
#6 0xffffffff80566aca at fork_exit+0x12a
#7 0xffffffff8085d18e at fork_trampoline+0xe
0xffffff0007d43270: tag zfs, type VREG
usecount 0, writecount 0, refcount 1 mountedhere 0
flags (VI_DOOMED)
v_object 0xffffff0007d2e870 ref 0 pages 0
lock type zfs: EXCL by thread 0xffffff000262dae0 (pid 1699)
#0 0xffffffff80578f77 at __lockmgr_args+0x777
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff806231db at vnlru_free+0x20b
#4 0xffffffff806235df at getnewvnode+0x24f
#5 0xffffffff810a7533 at zfs_znode_cache_constructor+0x43
#6 0xffffffff810a9248 at zfs_znode_alloc+0x38
#7 0xffffffff810a96d9 at zfs_mknode+0x259
#8 0xffffffff810cda21 at zfs_freebsd_create+0x661
#9 0xffffffff808cbc43 at VOP_CREATE_APV+0xb3
#10 0xffffffff8062d553 at vn_open_cred+0x473
#11 0xffffffff8062b569 at kern_openat+0x179
#12 0xffffffff80876b48 at syscall+0x118
#13 0xffffffff8085cf91 at Xfast_syscall+0xe1
0xffffff0007d61c30: tag zfs, type VREG
usecount 0, writecount 0, refcount 1 mountedhere 0
flags (VI_DOOMED)
v_object 0xffffff0007d2e6c0 ref 0 pages 0
lock type zfs: EXCL by thread 0xffffff000250d740 (pid 17)
#0 0xffffffff80578f77 at __lockmgr_args+0x777
#1 0xffffffff80610a59 at vop_stdlock+0x39
#2 0xffffffff808caedb at VOP_LOCK1_APV+0x9b
#3 0xffffffff806231db at vnlru_free+0x20b
#4 0xffffffff80623bce at vnlru_proc+0x58e
#5 0xffffffff80566aca at fork_exit+0x12a
#6 0xffffffff8085d18e at fork_trampoline+0xe
panic: from debugger
cpuid =3D 0
Uptime: 33m28s
#=20
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?E04F91AA-B2C4-4166-A24A-74F1BEF01519>