Date: Mon, 14 May 2007 02:53:21 +0300 From: Nikolay Pavlov <quetzal@zone3000.net> To: freebsd-fs@freebsd.org Subject: ZFS lock order reversal. Message-ID: <20070513235321.GA21268@zone3000.net>
next in thread | raw e-mail | index | archive | help
Hi. I am getting this lock order reversal while compiling ports.
I am using a simple disk pool.
quetzal@orion:~> zpool list <932>
NAME SIZE USED AVAIL CAP HEALTH ALTROOT
pool 11,6G 1,37G 10,3G 11% ONLINE -
quetzal@orion:~> zpool status <933>
pool: pool
state: ONLINE
scrub: none requested
config:
NAME STATE READ WRITE CKSUM
pool ONLINE 0 0 0
ad0s3 ONLINE 0 0 0
errors: No known data errors
quetzal@orion:~> zfs list <935>
NAME USED AVAIL REFER MOUNTPOINT
pool 1,37G 10,1G 18K /pool
pool/distfiles 904M 10,1G 904M /usr/ports/distfiles
pool/local 158M 10,1G 158M /usr/local
pool/packages 55,0M 10,1G 55,0M /usr/ports/packages
pool/ports 285M 10,1G 285M /usr/ports
May 14 02:46:02 orion kernel: lock order reversal:
May 14 02:46:02 orion kernel: 1st 0xc737da20 zfs:&dr->dt.di.dr_mtx (zfs:&dr->dt.di.dr_mtx) @ /usr/src/sys/modules/zfs/../../contrib/opensolaris/uts/common/
fs/zfs/dbuf.c:1866
May 14 02:46:02 orion kernel: 2nd 0xd111bcd0 zfs:&db->db_mtx (zfs:&db->db_mtx) @ /usr/src/sys/modules/zfs/../../contrib/opensolaris/uts/common/fs/zfs/dbuf.
c:1888
May 14 02:46:02 orion kernel: KDB: stack backtrace:
May 14 02:46:02 orion kernel: db_trace_self_wrapper(c0962c19) at db_trace_self_wrapper+0x25
May 14 02:46:02 orion kernel: kdb_backtrace(0,ffffffff,c0a72c70,c0a72400,c0a0de2c,...) at kdb_backtrace+0x29
May 14 02:46:02 orion kernel: witness_checkorder(d111bcd0,9,c85b1f8c,760) at witness_checkorder+0x586
May 14 02:46:02 orion kernel: _sx_xlock(d111bcd0,c85b1f8c,760,c837bd20,637,...) at _sx_xlock+0x52
May 14 02:46:02 orion kernel: dbuf_sync_list(c737da38,d0020e80,0,7,c8515000,...) at dbuf_sync_list+0x15b
May 14 02:46:02 orion kernel: dbuf_sync_list(c4c559e0,d0020e80,257,3c,0,...) at dbuf_sync_list+0xde
May 14 02:46:02 orion kernel: dnode_sync(c4c55910,d0020e80,c4b32898,c4c55910,30,...) at dnode_sync+0x3a8
May 14 02:46:02 orion kernel: dmu_objset_sync_dnodes(c8515000,d0020e80,637,0,ca1a5ac8,...) at dmu_objset_sync_dnodes+0x29
May 14 02:46:02 orion kernel: dmu_objset_sync(c4b32800,c486a000,d0020e80,c476a800,0,...) at dmu_objset_sync+0x11d
May 14 02:46:02 orion kernel: dsl_pool_sync(c43b8200,637,0,c476a800,637,...) at dsl_pool_sync+0x13e
May 14 02:46:02 orion kernel: spa_sync(c476a800,637,0,c43b82ac,c85b592d,...) at spa_sync+0x33f
May 14 02:46:02 orion kernel: txg_sync_thread(c43b8200,e6a1ed38) at txg_sync_thread+0x183
May 14 02:46:02 orion kernel: fork_exit(c8584924,c43b8200,e6a1ed38) at fork_exit+0xac
May 14 02:46:02 orion kernel: fork_trampoline() at fork_trampoline+0x8
May 14 02:46:02 orion kernel: --- trap 0, eip = 0, esp = 0xe6a1ed70, ebp = 0 ---
May 14 02:46:02 orion kernel: lock order reversal:
May 14 02:46:02 orion kernel: 1st 0xc4908520 zfs:&dr->dt.di.dr_mtx (zfs:&dr->dt.di.dr_mtx) @ /usr/src/sys/modules/zfs/../../contrib/opensolaris/uts/common/
fs/zfs/dbuf.c:1866
May 14 02:46:02 orion kernel: 2nd 0xd18faaa0 zfs:&db->db_mtx (zfs:&db->db_mtx) @ /usr/src/sys/modules/zfs/../../contrib/opensolaris/uts/common/fs/zfs/dbuf.
c:1888
May 14 02:46:02 orion kernel: KDB: stack backtrace:
May 14 02:46:02 orion kernel: db_trace_self_wrapper(c0962c19) at db_trace_self_wrapper+0x25
May 14 02:46:02 orion kernel: kdb_backtrace(0,ffffffff,c0a72108,c0a72400,c0a0de2c,...) at kdb_backtrace+0x29
May 14 02:46:02 orion kernel: witness_checkorder(d18faaa0,9,c85b1f8c,760) at witness_checkorder+0x586
May 14 02:46:02 orion kernel: _sx_xlock(d18faaa0,c85b1f8c,760,c81c2620,637,...) at _sx_xlock+0x52
May 14 02:46:02 orion kernel: dbuf_sync_list(c4908538,c9513380,c58bb520,637,0,...) at dbuf_sync_list+0x15b
May 14 02:46:02 orion kernel: dbuf_sync_list(c793a810,c9513380,257,3c,0,...) at dbuf_sync_list+0xde
May 14 02:46:02 orion kernel: dnode_sync(c793a740,c9513380,c4b32898,c793a740,30,...) at dnode_sync+0x3a8
May 14 02:46:02 orion kernel: dmu_objset_sync_dnodes(c8515000,c9513380,637,0,c7e2f678,...) at dmu_objset_sync_dnodes+0x29
May 14 02:46:02 orion kernel: dmu_objset_sync(c4b32800,c4982000,c9513380,c476a800,0,...) at dmu_objset_sync+0x11d
May 14 02:46:02 orion kernel: dsl_pool_sync(c43b8200,637,0,c476a800,637,...) at dsl_pool_sync+0x13e
May 14 02:46:02 orion kernel: spa_sync(c476a800,637,0,c43b82ac,c85b592d,...) at spa_sync+0x33f
May 14 02:46:02 orion kernel: txg_sync_thread(c43b8200,e6a1ed38) at txg_sync_thread+0x183
May 14 02:46:02 orion kernel: fork_exit(c8584924,c43b8200,e6a1ed38) at fork_exit+0xac
May 14 02:46:02 orion kernel: fork_trampoline() at fork_trampoline+0x8
May 14 02:46:02 orion kernel: --- trap 0, eip = 0, esp = 0xe6a1ed70, ebp = 0 ---
vfs.zfs.dnlc.enable: 0
vfs.zfs.dnlc.max_nentries: 68506
vfs.zfs.dnlc.nentries: 0
vfs.zfs.dnlc.ncsize: 34253
vfs.zfs.arc_min: 16777216
vfs.zfs.arc_max: 167772160
vfs.zfs.mdcomp_disable: 0
vfs.zfs.prefetch_disable: 0
vfs.zfs.zio.taskq_threads: 0
vfs.zfs.recover: 0
vfs.zfs.vdev.cache.size: 10485760
vfs.zfs.vdev.cache.max: 16384
vfs.zfs.cache_flush_disable: 0
vfs.zfs.zil_disable: 0
vfs.zfs.debug: 0
kstat.zfs.misc.dnlcstats.hits: 0
kstat.zfs.misc.dnlcstats.misses: 0
kstat.zfs.misc.dnlcstats.negative_cache_hits: 0
kstat.zfs.misc.dnlcstats.enters: 0
kstat.zfs.misc.dnlcstats.double_enters: 0
kstat.zfs.misc.dnlcstats.purge_total_entries: 0
kstat.zfs.misc.dnlcstats.purge_all: 0
kstat.zfs.misc.dnlcstats.purge_vp: 0
kstat.zfs.misc.dnlcstats.purge_vfs: 0
kstat.zfs.misc.dnlcstats.purge_fs1: 0
kstat.zfs.misc.dnlcstats.pick_free: 0
kstat.zfs.misc.dnlcstats.pick_heuristic: 0
kstat.zfs.misc.dnlcstats.pick_last: 0
kstat.zfs.misc.arcstats.hits: 376113
kstat.zfs.misc.arcstats.misses: 84432
kstat.zfs.misc.arcstats.demand_data_hits: 232938
kstat.zfs.misc.arcstats.demand_data_misses: 3687
kstat.zfs.misc.arcstats.demand_metadata_hits: 123332
kstat.zfs.misc.arcstats.demand_metadata_misses: 75398
kstat.zfs.misc.arcstats.prefetch_data_hits: 296
kstat.zfs.misc.arcstats.prefetch_data_misses: 184
kstat.zfs.misc.arcstats.prefetch_metadata_hits: 19547
kstat.zfs.misc.arcstats.prefetch_metadata_misses: 5163
kstat.zfs.misc.arcstats.mru_hits: 44441
kstat.zfs.misc.arcstats.mru_ghost_hits: 1065
kstat.zfs.misc.arcstats.mfu_hits: 311926
kstat.zfs.misc.arcstats.mfu_ghost_hits: 5535
kstat.zfs.misc.arcstats.deleted: 245988
kstat.zfs.misc.arcstats.recycle_miss: 272594
kstat.zfs.misc.arcstats.mutex_miss: 129
kstat.zfs.misc.arcstats.evict_skip: 4839885
kstat.zfs.misc.arcstats.hash_elements: 6228
kstat.zfs.misc.arcstats.hash_elements_max: 13979
kstat.zfs.misc.arcstats.hash_collisions: 65501
kstat.zfs.misc.arcstats.hash_chains: 979
kstat.zfs.misc.arcstats.hash_chain_max: 6
kstat.zfs.misc.arcstats.p: 36156672
kstat.zfs.misc.arcstats.c: 46258688
kstat.zfs.misc.arcstats.c_min: 16777216
kstat.zfs.misc.arcstats.c_max: 167772160
kstat.zfs.misc.arcstats.size: 43229184
quetzal@orion:~> uname -a
FreeBSD orion.zone3000.net 7.0-CURRENT FreeBSD 7.0-CURRENT #0: Fri May 11 01:37:28 EEST 2007
root@orion.zone3000.net:/usr/obj/usr/src/sys/GENERIC i386
--
======================================================================
- Best regards, Nikolay Pavlov. <<<-----------------------------------
======================================================================
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20070513235321.GA21268>