Date: Wed, 27 Jul 2011 12:16:50 +0100 From: "Steven Hartland" <killing@multiplay.co.uk> To: <freebsd-fs@FreeBSD.ORG> Subject: zfs process hang on pool access Message-ID: <A14F1C768A41483C876AD77502A864D6@multiplay.co.uk>
next in thread | raw e-mail | index | archive | help
Got a machine which is hung accessing a specific zfs pool, other
volumes seem unaffected, there's no errors showing on zpool status and
no errors in /var/log/messages
Processes seem to be hung in a variaty of stats including:-
STOP, zfs, zio->i, db->db, tx->tx
zfs list also hangs.
Here's some procstat -k -k <pid> from some hung processes and the output
from zfs-stats -a
The machine is running 8.2-RELEASE
procstat -k -k 94003
PID TID COMM TDNAME KSTACK
94003 100341 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 100417 java initial thread mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_wait+0x72a __umtx_op_wait+0x5e syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 100459 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 100536 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 100544 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19
_cv_timedwait_sig+0x134 seltdwait+0x98 poll+0x2f8 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 100751 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 100925 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101268 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101319 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101417 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101486 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101498 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101555 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_wait+0x72a __umtx_op_wait_uint_private+0x64 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101563 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101565 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101566 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101804 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101897 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101971 java - mi_switch+0x176 sleepq_wait+0x42 _sx_xlock_hard+0x305 _sx_xlock+0x4e
txg_hold_open+0x4f dmu_tx_assign+0x189 zfs_freebsd_create+0x2f2 VOP_CREATE_APV+0x31 vn_open_cred+0x4ab kern_openat+0x181
syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 101984 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 102164 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _cv_wait_sig+0x128
seltdwait+0x110 poll+0x2f8 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 102627 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103546 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103636 java - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 zio_wait+0x61 dbuf_read+0x39a
dbuf_findbp+0xf7 dbuf_hold_impl+0xc2 dbuf_hold_level+0x1a dmu_tx_check_ioerr+0x52 dmu_tx_count_write+0x297 dmu_tx_hold_write+0x4a
zfs_freebsd_write+0x397 VOP_WRITE_APV+0xb2 vn_write+0x2d7 dofilewrite+0x8b kern_writev+0x60 write+0x55 syscallenter+0x1e5
94003 103676 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103728 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103748 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103749 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103750 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
94003 103751 java - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 zio_wait+0x61 dbuf_read+0x39a
dbuf_findbp+0xf7 dbuf_hold_impl+0xc2 dbuf_hold_level+0x1a dmu_tx_check_ioerr+0x52 dmu_tx_count_write+0x297 dmu_tx_hold_write+0x4a
zfs_freebsd_write+0x397 VOP_WRITE_APV+0xb2 vn_write+0x2d7 dofilewrite+0x8b kern_writev+0x60 write+0x55 syscallenter+0x1e5
procstat -k -k 39568
PID TID COMM TDNAME KSTACK
39568 100303 find - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 zio_wait+0x61 dbuf_read+0x39a
dmu_buf_hold+0xcc zap_lockdir+0x55 zap_cursor_retrieve+0x194 zfs_freebsd_readdir+0x2b6 kern_getdirentries+0x217 getdirentries+0x23
syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
zfs-stats -a
------------------------------------------------------------------------
ZFS Subsystem Report Wed Jul 27 10:57:12 2011
------------------------------------------------------------------------
System Information:
Kernel Version: 802000 (osreldate)
Hardware Platform: amd64
Processor Architecture: amd64
FreeBSD 8.2-RELEASE #0: Fri Mar 18 10:58:44 UTC 2011 root
10:57AM up 112 days, 21 mins, 1 user, load averages: 0.56, 0.39, 0.34
------------------------------------------------------------------------
System Memory Statistics:
Physical Memory: 24555.62M
Kernel Memory: 2076.86M
DATA: 99.50% 2066.54M
TEXT: 0.50% 10.32M
------------------------------------------------------------------------
ZFS pool information:
Storage pool Version (spa): 15
Filesystem Version (zpl): 4
------------------------------------------------------------------------
ARC Misc:
Deleted: 122887476
Recycle Misses: 2985802
Mutex Misses: 51968
Evict Skips: 51968
ARC Size:
Current Size (arcsize): 12.50% 2847.37M
Target Size (Adaptive, c): 12.50% 2847.48M
Min Size (Hard Limit, c_min): 12.50% 2847.48M
Max Size (High Water, c_max): ~8:1 22779.81M
ARC Size Breakdown:
Recently Used Cache Size (p): 29.47% 839.09M
Freq. Used Cache Size (c-p): 70.53% 2008.38M
ARC Hash Breakdown:
Elements Max: 1093246
Elements Current: 34.92% 381795
Collisions: 496215463
Chain Max: 11
Chains: 86474
ARC Eviction Statistics:
Evicts Total: 2581704887296
Evicts Eligible for L2: 93.73% 2419921712128
Evicts Ineligible for L2: 6.27% 161783175168
Evicts Cached to L2: 0
ARC Efficiency:
Cache Access Total: 17042848480
Cache Hit Ratio: 99.85% 17017691729
Cache Miss Ratio: 0.15% 25156751
Actual Hit Ratio: 88.30% 15049590290
Data Demand Efficiency: 99.90%
Data Prefetch Efficiency: 86.74%
CACHE HITS BY CACHE LIST:
Anonymously Used: 11.48% 1952878601
Most Recently Used (mru): 5.30% 901233481
Most Frequently Used (mfu): 83.14% 14148356809
MRU Ghost (mru_ghost): 0.03% 5560514
MFU Ghost (mfu_ghost): 0.06% 9662324
CACHE HITS BY DATA TYPE:
Demand Data: 65.68% 11176417108
Prefetch Data: 0.26% 44828929
Demand Metadata: 6.89% 1173279354
Prefetch Metadata: 27.17% 4623166338
CACHE MISSES BY DATA TYPE:
Demand Data: 44.65% 11232287
Prefetch Data: 27.24% 6852876
Demand Metadata: 23.17% 5829877
Prefetch Metadata: 4.94% 1241711
------------------------------------------------------------------------
VDEV Cache Summary:
Access Total: 6646743
Hits Ratio: 67.29% 4472518
Miss Ratio: 32.71% 2174225
Delegations: 350939
------------------------------------------------------------------------
File-Level Prefetch Stats (DMU):
DMU Efficiency:
Access Total: 119890457768
Hit Ratio: 91.27% 109423001264
Miss Ratio: 8.73% 10467456504
Colinear Access Total: 10467456504
Colinear Hit Ratio: 0.01% 632312
Colinear Miss Ratio: 99.99% 10466824192
Stride Access Total: 107333142359
Stride Hit Ratio: 99.99% 107326197075
Stride Miss Ratio: 0.01% 6945284
DMU misc:
Reclaim successes: 2967491512
Reclaim failures: 7499332680
Stream resets: 58396
Stream noresets: 1273657946
Bogus streams: 0
------------------------------------------------------------------------
ZFS Tunable (sysctl):
kern.maxusers=384
vfs.zfs.l2c_only_size=0
vfs.zfs.mfu_ghost_data_lsize=495015424
vfs.zfs.mfu_ghost_metadata_lsize=96888320
vfs.zfs.mfu_ghost_size=591903744
vfs.zfs.mfu_data_lsize=17817088
vfs.zfs.mfu_metadata_lsize=286173184
vfs.zfs.mfu_size=563621376
vfs.zfs.mru_ghost_data_lsize=1540046336
vfs.zfs.mru_ghost_metadata_lsize=849328128
vfs.zfs.mru_ghost_size=2389374464
vfs.zfs.mru_data_lsize=232126464
vfs.zfs.mru_metadata_lsize=118643712
vfs.zfs.mru_size=531284992
vfs.zfs.anon_data_lsize=0
vfs.zfs.anon_metadata_lsize=0
vfs.zfs.anon_size=68533248
vfs.zfs.l2arc_norw=1
vfs.zfs.l2arc_feed_again=1
vfs.zfs.l2arc_noprefetch=0
vfs.zfs.l2arc_feed_min_ms=200
vfs.zfs.l2arc_feed_secs=1
vfs.zfs.l2arc_headroom=2
vfs.zfs.l2arc_write_boost=8388608
vfs.zfs.l2arc_write_max=8388608
vfs.zfs.arc_meta_limit=5971591168
vfs.zfs.arc_meta_used=2665786824
vfs.zfs.mdcomp_disable=0
vfs.zfs.arc_min=2985795584
vfs.zfs.arc_max=23886364672
vfs.zfs.zfetch.array_rd_sz=1048576
vfs.zfs.zfetch.block_cap=256
vfs.zfs.zfetch.min_sec_reap=2
vfs.zfs.zfetch.max_streams=8
vfs.zfs.prefetch_disable=0
vfs.zfs.check_hostid=1
vfs.zfs.recover=0
vfs.zfs.txg.write_limit_override=0
vfs.zfs.txg.synctime=5
vfs.zfs.txg.timeout=30
vfs.zfs.scrub_limit=10
vfs.zfs.vdev.cache.bshift=16
vfs.zfs.vdev.cache.size=10485760
vfs.zfs.vdev.cache.max=16384
vfs.zfs.vdev.aggregation_limit=131072
vfs.zfs.vdev.ramp_rate=2
vfs.zfs.vdev.time_shift=6
vfs.zfs.vdev.min_pending=4
vfs.zfs.vdev.max_pending=10
vfs.zfs.cache_flush_disable=0
vfs.zfs.zil_disable=0
vfs.zfs.zio.use_uma=0
vfs.zfs.version.zpl=4
vfs.zfs.version.spa=15
vfs.zfs.version.dmu_backup_stream=1
vfs.zfs.version.dmu_backup_header=2
vfs.zfs.version.acl=1
vfs.zfs.debug=0
vfs.zfs.super_owner=0
vm.kmem_size=24960106496
vm.kmem_size_scale=1
vm.kmem_size_min=0
vm.kmem_size_max=329853485875
------------------------------------------------------------------------
procstat -k -k 36541
PID TID COMM TDNAME KSTACK
36541 100971 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 101251 java initial thread mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_wait+0x72a __umtx_op_wait+0x5e syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 101591 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 101648 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 101781 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 101798 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 101993 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102037 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102059 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102191 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_wait+0x72a __umtx_op_wait_uint_private+0x64 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102208 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102210 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102221 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102253 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102279 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102298 java - mi_switch+0x176 sleepq_wait+0x42 _sx_xlock_hard+0x305 _sx_xlock+0x4e
txg_hold_open+0x4f dmu_tx_assign+0x189 zfs_freebsd_create+0x2f2 VOP_CREATE_APV+0x31 vn_open_cred+0x4ab kern_openat+0x181
syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102426 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102594 java - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 zio_wait+0x61
dmu_buf_hold_array_by_dnode+0x217 dmu_buf_hold_array+0x6a dmu_read_uio+0x3f zfs_freebsd_read+0x5d3 vn_read+0x2cc dofileread+0xa1
kern_readv+0x60 read+0x55 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 102775 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 103293 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 103790 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 103791 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36541 103797 java - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 zio_wait+0x61 dbuf_read+0x39a
dmu_buf_hold+0xcc zap_lockdir+0x55 zap_lookup_norm+0x45 zap_lookup+0x2e zfs_dirent_lock+0x534 zfs_dirlook+0x69 zfs_lookup+0x26b
zfs_freebsd_lookup+0x81 vfs_cache_lookup+0xf0 VOP_LOOKUP_APV+0x40 lookup+0x452 namei+0x53a kern_statat_vnhook+0x8f
36541 103809 java - mi_switch+0x176 sleepq_wait+0x42 __lockmgr_args+0x75a vop_stdlock+0x39
VOP_LOCK1_APV+0x46 _vn_lock+0x47 vget+0x70 cache_lookup+0x50f vfs_cache_lookup+0xc0 VOP_LOOKUP_APV+0x40 lookup+0x452 namei+0x53a
vn_open_cred+0x3ac kern_openat+0x181 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
procstat -k -k 36732
PID TID COMM TDNAME KSTACK
36732 100369 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100790 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100794 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100830 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100853 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100873 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100957 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 100977 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 101124 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_wait+0x72a __umtx_op_wait_uint_private+0x64 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 101130 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 101190 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 101326 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_cv_wait+0x871 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 102146 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 102257 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 102316 java - mi_switch+0x176 sleepq_wait+0x42 _sx_xlock_hard+0x305 _sx_xlock+0x4e
txg_hold_open+0x4f dmu_tx_assign+0x189 zfs_freebsd_create+0x2f2 VOP_CREATE_APV+0x31 vn_open_cred+0x4ab kern_openat+0x181
syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 102484 java - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 dmu_buf_hold_array_by_dnode+0x28f
dmu_buf_hold_array+0x6a dmu_read_uio+0x3f zfs_freebsd_read+0x5d3 vn_read+0x2cc dofileread+0xa1 kern_readv+0x60 read+0x55
syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 102943 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _cv_wait_sig+0x128
seltdwait+0x110 poll+0x2f8 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 103102 java initial thread mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_wait_sig+0x16 _sleep+0x269
do_wait+0x72a __umtx_op_wait+0x5e syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 103255 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19 _sleep+0x1b1
do_cv_wait+0x640 __umtx_op_cv_wait+0x5c syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 103292 java - mi_switch+0x176 sleepq_catch_signals+0x31c sleepq_timedwait_sig+0x19
_cv_timedwait_sig+0x134 seltdwait+0x98 poll+0x2f8 syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
36732 103481 java - mi_switch+0x176 sleepq_wait+0x42 _cv_wait+0x129 dmu_buf_hold_array_by_dnode+0x28f
dmu_buf_hold_array+0x6a dmu_read_uio+0x3f zfs_freebsd_read+0x5d3 vn_read+0x2cc dofileread+0xa1 kern_readv+0x60 read+0x55
syscallenter+0x1e5 syscall+0x4b Xfast_syscall+0xe2
================================================
This e.mail is private and confidential between Multiplay (UK) Ltd. and the person or entity to whom it is addressed. In the event of misdirection, the recipient is prohibited from using, copying, printing or otherwise disseminating it or any information contained in it.
In the event of misdirection, illegible or incomplete transmission please telephone +44 845 868 1337
or return the E.mail to postmaster@multiplay.co.uk.
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?A14F1C768A41483C876AD77502A864D6>