Date: Mon, 03 Mar 2014 03:03:30 -0800 (PST) From: Anton Shterenlikht <mexas@bris.ac.uk> To: FreeBSD-gnats-submit@freebsd.org Subject: ia64/187219: panic: 11.0-CURRENT #20 r260914: Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks Message-ID: <201403031103.s23B3TBV001131@mech-as28.men.bris.ac.uk> Resent-Message-ID: <201403031110.s23BA0bL011014@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 187219
>Category: ia64
>Synopsis: panic: 11.0-CURRENT #20 r260914: Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: freebsd-ia64
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Mon Mar 03 11:10:00 UTC 2014
>Closed-Date:
>Last-Modified:
>Originator: Anton Shterenlikht
>Release: FreeBSD 11.0-CURRENT ia64
>Organization:
University of Bristol
>Environment:
System: FreeBSD mech-as28.men.bris.ac.uk 11.0-CURRENT FreeBSD 11.0-CURRENT #20 r260914: Fri Feb 28 14:36:02 GMT 2014 root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV ia64
>Description:
With textdump set up, I collected these files:
# cat info.last
Dump header from device /dev/da2p1
Architecture: ia64
Architecture Version: 1
Dump Length: 67584B (0 MB)
Blocksize: 512
Dumptime: Mon Mar 3 10:35:01 2014
Hostname: mech-as28.men.bris.ac.uk
Magic: FreeBSD Text Dump
Version String: FreeBSD 11.0-CURRENT #20 r260914: Fri Feb 28 14:36:02 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV
Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks
Dump Parity: 2806468409
Bounds: 8
Dump Status: good
#
# cat panic.txt
deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks
#
# cat version.txt
FreeBSD 11.0-CURRENT #20 r260914: Fri Feb 28 14:36:02 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV
#
# cat config.txt
options CONFIG_AUTOGENERATED
ident ZEEV
machine ia64
cpu ITANIUM2
makeoptions MODULES_OVERRIDE=geom/geom_part opensolaris zfs acl_nfs4 fdescfs
makeoptions DEBUG=-g
options USB_DEBUG
options DEBUG_VFS_LOCKS
options DEBUG_LOCKS
options TEXTDUMP_VERBOSE
options TEXTDUMP_PREFERRED
options MALLOC_DEBUG_MAXZONES=8
options WITNESS_SKIPSPIN
options WITNESS
options INVARIANT_SUPPORT
options INVARIANTS
options DEADLKRES
options GDB
options DDB
options KDB_TRACE
options KDB
options _KPOSIX_PRIORITY_SCHEDULING
options QUOTA
options UFS_GJOURNAL
options UFS_DIRHASH
options UFS_ACL
options SYSVSHM
options SYSVSEM
options SYSVMSG
options STACK
options SOFTUPDATES
options SMP
options SCTP
options SCSI_DELAY=5000
options SCHED_ULE
options PSEUDOFS
options PROCFS
options PRINTF_BUFR_SIZE=128
options PREEMPTION
options P1003_1B_SEMAPHORES
options NFS_ROOT
options NFSD
options NFSLOCKD
options NFSCL
options MSDOSFS
options MD_ROOT
options MAC
options KTRACE
options INET6
options INET
options INCLUDE_CONFIG_FILE
options GEOM_LABEL
options FFS
options TMPFS
options NULLFS
options FDESCFS
options COMPAT_FREEBSD7
options CD9660
options CAPABILITIES
options CAPABILITY_MODE
options AUDIT
options NEW_PCIB
options GEOM_PART_MBR
options GEOM_PART_GPT
options GEOM_PART_BSD
device acpi
device io
device mem
device uart_ns8250
device miibus
device pci
device scbus
device usb
device isp
device ispfw
device mpt
device cd
device da
device pass
device sa
device ehci
device ohci
device uhci
device umass
device em
device ether
device faith
device firmware
device gif
device loop
device md
device puc
device random
device tun
device uart
device vlan
# cat msgbuf.txt
GDB: debug ports: uart
GDB: current port: uart
KDB: debugger backends: ddb gdb
KDB: current backend: ddb
Copyright (c) 1992-2014 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 11.0-CURRENT #20 r260914: Fri Feb 28 14:36:02 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV ia64
gcc version 4.2.1 20070831 patched [FreeBSD]
WARNING: WITNESS option enabled, expect reduced performance.
CPU: Madison II (1600 MHz Itanium 2)
Origin = "GenuineIntel" Revision = 2
Features = 0x1<LB>
real memory = 10737418240 (10240 MB)
avail memory = 10494246912 (10008 MB)
FPSWA Revision = 0x10012, Entry = 0xe0000040ffcc4050
FreeBSD/SMP: Multiprocessor System Detected: 2 CPUs
cpu0: ACPI Id=0, SAPIC Id=0, SAPIC Eid=0 (BSP)
cpu1: ACPI Id=1, SAPIC Id=1, SAPIC Eid=0
random: <Software, Yarrow> initialized
Event timer "ITC" frequency 1600000000 Hz quality 1000
ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe0Block: 32/16 (20130823/tbfadt-603)
ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe1Block: 32/16 (20130823/tbfadt-603)
acpi0: <HP> on motherboard
acpi0: Power Button (fixed)
acpi0: Sleep Button (fixed)
cpu0: <ACPI CPU> on acpi0
cpu1: <ACPI CPU> on acpi0
Timecounter "ACPI-safe" frequency 3579545 Hz quality 850
acpi_timer0: <32-bit timer at 3.579545MHz> iomem 0xff5c1004-0xff5c1007 on acpi0
acpi_tz0: <Thermal Zone> on acpi0
pcib0: <ACPI Host-PCI bridge> on acpi0
pcib0: could not evaluate _ADR - AE_NOT_FOUND
pci0: <ACPI PCI bus> on pcib0
ohci0: <NEC uPD 9210 USB controller> mem 0x80002000-0x80002fff irq 16 at device 1.0 on pci0
usbus0 on ohci0
ohci1: <NEC uPD 9210 USB controller> mem 0x80001000-0x80001fff irq 17 at device 1.1 on pci0
usbus1 on ohci1
ehci0: <NEC uPD 720100 USB 2.0 controller> mem 0x80000000-0x800000ff irq 18 at device 1.2 on pci0
usbus2: EHCI version 0.95
usbus2 on ehci0
pci0: <mass storage, ATA> at device 2.0 (no driver attached)
pcib1: <ACPI Host-PCI bridge> on acpi0
pcib1: could not evaluate _ADR - AE_NOT_FOUND
pci32: <ACPI PCI bus> on pcib1
mpt0: <LSILogic 1030 Ultra4 Adapter> port 0x2100-0x21ff mem 0x903a0000-0x903bffff,0x90380000-0x9039ffff irq 27 at device 1.0 on pci32
mpt0: MPI Version=1.2.12.0
mpt1: <LSILogic 1030 Ultra4 Adapter> port 0x2000-0x20ff mem 0x90360000-0x9037ffff,0x90340000-0x9035ffff irq 28 at device 1.1 on pci32
mpt1: MPI Version=1.2.12.0
em0: <Intel(R) PRO/1000 Legacy Network Connection 1.0.6> port 0x2240-0x227f mem 0x90320000-0x9033ffff,0x90280000-0x902fffff irq 29 at device 2.0 on pci32
em0: Ethernet address: 00:13:21:5b:05:1c
em1: <Intel(R) PRO/1000 Legacy Network Connection 1.0.6> port 0x2200-0x223f mem 0x90300000-0x9031ffff irq 30 at device 2.1 on pci32
em1: Ethernet address: 00:13:21:5b:05:1d
pcib2: <ACPI Host-PCI bridge> on acpi0
pcib2: could not evaluate _ADR - AE_NOT_FOUND
pci64: <ACPI PCI bus> on pcib2
pcib3: <ACPI Host-PCI bridge> on acpi0
pcib3: could not evaluate _ADR - AE_NOT_FOUND
pci96: <ACPI PCI bus> on pcib3
pcib4: <ACPI Host-PCI bridge> on acpi0
pcib4: could not evaluate _ADR - AE_NOT_FOUND
pci128: <ACPI PCI bus> on pcib4
pcib5: <ACPI Host-PCI bridge> on acpi0
pcib5: could not evaluate _ADR - AE_NOT_FOUND
pci192: <ACPI PCI bus> on pcib5
isp0: <Qlogic ISP 2422 PCI FC-AL Adapter> port 0xc000-0xc0ff mem 0xe0040000-0xe0040fff irq 71 at device 1.0 on pci192
pcib6: <ACPI Host-PCI bridge> on acpi0
pcib6: could not evaluate _ADR - AE_NOT_FOUND
pci224: <ACPI PCI bus> on pcib6
uart0: <HP Auxiliary Diva Serial Port> mem 0xf8051000-0xf805100f irq 82 at device 1.0 on pci224
puc0: <HP Diva Serial [GSP] Multiport UART - Everest SP2> mem 0xf8050000-0xf8050fff,0xf8020000-0xf803ffff irq 82 at device 1.1 on pci224
uart1: <Non-standard ns8250 class UART with FIFOs> at port 1 on puc0
uart1: console (9600,n,8,1)
uart2: <Non-standard ns8250 class UART with FIFOs> at port 2 on puc0
uart3: <Non-standard ns8250 class UART with FIFOs> at port 3 on puc0
vgapci0: <VGA-compatible display> port 0xe000-0xe0ff mem 0xf0000000-0xf7ffffff,0xf8040000-0xf804ffff at device 2.0 on pci224
vgapci0: Boot video device
uart4: <16550 or compatible> iomem 0xff5e0000-0xff5e0007 irq 34 on acpi0
uart5: <16550 or compatible> iomem 0xff5e2000-0xff5e2007 irq 35 on acpi0
uart5: debug port (9600,n,8,1)
ZFS filesystem version: 5
ZFS storage pool version: features support (5000)
Timecounters tick every 1.000 msec
usbus0: 12Mbps Full Speed USB v1.0
usbus1: 12Mbps Full Speed USB v1.0
usbus2: 480Mbps High Speed USB v2.0
ugen0.1: <NEC> at usbus0
ugen1.1: <NEC> at usbus1
uhub0: <NEC OHCI root HUB, class 9/0, rev 1.00/1.00, addr 1> on usbus0
uhub1: <NEC OHCI root HUB, class 9/0, rev 1.00/1.00, addr 1> on usbus1
ugen2.1: <NEC> at usbus2
uhub2: <NEC EHCI root HUB, class 9/0, rev 2.00/1.00, addr 1> on usbus2
random: unblocking device.
uhub1: 2 ports with 2 removable, self powered
uhub0: 3 ports with 3 removable, self powered
uhub2: 5 ports with 5 removable, self powered
da0 at mpt0 bus 0 scbus0 target 0 lun 0
da0: <HP 73.4G ST373454LC HPC2> Fixed Direct Access SCSI-3 device
da0: Serial Number 3KP05D0W00007532H5SA
da0: 320.000MB/s transfers (160.000MHz, offset 63, 16bit)
da0: Command Queueing enabled
da0: 70007MB (143374738 512 byte sectors: 255H 63S/T 8924C)
da1 at isp0 bus 0 scbus2 target 0 lun 1
da1: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da1: Serial Number P56350GX3RX04Z
da1: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da1: Command Queueing enabled
da1: 69460MB (142255575 512 byte sectors: 255H 63S/T 8855C)
da2 at isp0 bus 0 scbus2 target 0 lun 2
da2: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da2: Serial Number P56350GX3RX04Z
da2: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da2: Command Queueing enabled
da2: 69460MB (142255575 512 byte sectors: 255H 63S/T 8855C)
da3 at isp0 bus 0 scbus2 target 0 lun 5
da3: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da3: Serial Number P56350GX3RX04Z
da3: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da3: Command Queueing enabled
da3: 140011MB (286744185 512 byte sectors: 255H 63S/T 17849C)
pass1 at isp0 bus 0 scbus2 target 0 lun 0
pass1: <COMPAQ MSA1000 4.32> Fixed Storage Array SCSI-4 device
pass1: Serial Number P56350GX3RX04Z
pass1: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
pass1: Command Queueing enabled
WARNING: WITNESS option enabled, expect reduced performance.
Trying to mount root from ufs:/dev/da0p2 [rw]...
WARNING: / was not properly dismounted
<118>Setting hostuuid: ee652761-3dcc-11df-9074-0013215b051c.
<118>Setting hostid: 0x8968724b.
<118>Starting ddb.
<118>Entropy harvesting: interrupts ethernet point_to_point swi.
<118>Starting file system checks:
<118>/dev/da0p2: 560961 files, 3745925 used, 30870499 free (304931 frags, 3820696 blocks, 0.9% fragmentation)
<118>/dev/da1p1: 2133 files, 78964 used, 17144397 free (461 frags, 2142992 blocks, 0.0% fragmentation)
<118>/dev/da3p1: 6067 files, 1327969 used, 68099534 free (6214 frags, 8511665 blocks, 0.0% fragmentation)
<118>Mounting local file systems:.
<118>Writing entropy file:.
<118>Setting hostname: mech-as28.men.bris.ac.uk.
<118>Starting Network: lo0 em0 em1.
<118>lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
<118> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
<118> inet6 ::1 prefixlen 128
<118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
<118> inet 127.0.0.1 netmask 0xff000000
<118> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
<118>em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC>
<118> ether 00:13:21:5b:05:1c
<118> inet 137.222.187.28 netmask 0xffffff00 broadcast 137.222.187.255
<118> inet6 fe80::213:21ff:fe5b:51c%em0 prefixlen 64 scopeid 0x1
<118> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
<118> media: Ethernet autoselect
<118> status: no carrier
<118>em1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC>
<118> ether 00:13:21:5b:05:1d
<118> inet 10.10.10.14 netmask 0xffffff00 broadcast 10.10.10.255
<118> inet6 fe80::213:21ff:fe5b:51d%em1 prefixlen 64 scopeid 0x2
<118> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
<118> media: Ethernet autoselect (100baseTX <full-duplex>)
<118> status: active
<118>Starting devd.
<118>Starting pflogd:
<118>add net default: gateway 137.222.187.250
<118>add net fe80::: gateway ::1
<118>add net ff02::: gateway ::1
<118>add net ::ffff:0.0.0.0: gateway ::1
<118>add net ::0.0.0.0: gateway ::1
<118>ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib
<118>Creating and/or trimming log files.
<118>Starting syslogd.
<118>savecore: reboot after panic: deadlkres: possible deadlock detected for 0xe000000012424480, blocked for 1802103 ticks
<118>Mar 3 09:50:45 mech-as28 savecore: reboot after panic: deadlkres: possible deadlock detected for 0xe000000012424480, blocked for 1802103 ticks
<118>savecore: writing core to /var/crash/textdump.tar.7
<118>/var/crash/vmcore.7 not found
<118>Starting casperd.
lock order reversal:
1st 0xa00000009e060078 bufwait (bufwait) @ /usr/src/sys/kern/vfs_bio.c:3081
2nd 0xe000000010c38000 dirhash (dirhash) @ /usr/src/sys/ufs/ufs/ufs_dirhash.c:284
KDB: stack backtrace:
db_trace_self(0x9ffc0000000d89c0) at db_trace_self+0x40
db_trace_self_wrapper(0x9ffc0000005842c0) at db_trace_self_wrapper+0x70
kdb_backtrace(0x9ffc000000e4eba8, 0x9ffc0000005ba3e0) at kdb_backtrace+0xc0
_witness_debugger(0x1, 0x9ffc000000d4bdf8, 0x9ffc0000005bddc0, 0xb9d, 0x9ffc000000d799e8) at _witness_debugger+0x60
witness_checkorder(0xe000000010c38000, 0x9ffc000000d4b740, 0x9ffc000000d799e8, 0x11c, 0x0) at witness_checkorder+0x15c0
_sx_xlock(0xe000000010c38000, 0x0, 0x9ffc000000d799e8, 0x11c) at _sx_xlock+0x120
ufsdirhash_acquire(0xe0000000123c8f18, 0xe000000010c38000, 0x9ffc000000989250) at ufsdirhash_acquire+0x70
ufsdirhash_add(0xe0000000123c8f18, 0xa00000000036d098, 0x90c, 0xa00000000036d040, 0x9ffc000000991140, 0xda1) at ufsdirhash_add+0x20
ufs_direnter(0xe000000012689380, 0xe000000012694750, 0xa00000000036d098, 0xa00000000036d318, 0xa00000009e0609b0, 0x0) at ufs_direnter+0x1260
ufs_mkdir(0xa00000000036d370, 0xa00000000036d098, 0xe000000012689380) at ufs_mkdir+0x14f0
VOP_MKDIR_APV(0x9ffc000000e13a50, 0xa00000000036d370, 0x0, 0x9ffc000000d8a708, 0x9ffc000000677b70, 0x818, 0x9ffc000001022ba0) at VOP_MKDIR_APV+0x270
kern_mkdirat(0xe000000012636480, 0xffffffffffffff9c, 0x140804040, 0x0, 0x1c0) at kern_mkdirat+0x3a0
kern_mkdir(0xe000000012636480, 0x140804040, 0x0, 0x1c0) at kern_mkdir+0x40
sys_mkdir(0xe000000012636480, 0xa00000000036d4e8, 0x9ffc000000a40c80, 0x48d) at sys_mkdir+0x30
syscall(0xe0000000124464a0, 0x1c0, 0x9ffc000000a3ccb0, 0xe000000012636480, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
<118>Clearing /tmp (X related).
<118>Recovering vi editor sessions:.
<118>Updating motd:.
<118>Mounting late file systems:.
<118>Starting ntpd.
<118>Starting bsdstats.
<118>Posting monthly OS statistics to rpt.bsdstats.org
<118>Performing sanity check on sshd configuration.
<118>Starting sshd.
<118>Starting sendmail_submit.
<118>Starting sendmail_msp_queue.
<118>Starting cron.
<118>Local package initialization:cd: /var/portbuild/: No such file or directory
<118>.
<118>Starting inetd.
<118>Starting background file system checks in 60 seconds.
<118>
<118>Mon Mar 3 09:50:54 GMT 2014
<118>Mar 3 09:56:32 mech-as28 login: ROOT LOGIN (root) ON ttyu1
panic: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks
cpuid = 0
KDB: stack backtrace:
db_trace_self(0x9ffc0000000d89c0) at db_trace_self+0x40
db_trace_self_wrapper(0x9ffc0000005842c0) at db_trace_self_wrapper+0x70
kdb_backtrace(0x9ffc000000e4eba8, 0x9ffc00000050b420, 0x40c, 0x9ffc000001022ba0) at kdb_backtrace+0xc0
vpanic(0x9ffc000000d36760, 0xa000000000275518) at vpanic+0x260
panic(0x9ffc000000d36760, 0x9ffc000000d369b8, 0xe000000012079b00, 0x1b7ee8) at panic+0x80
deadlkres(0x9ffc000000d366f0, 0xe000000012079b00, 0x9ffc000000de9210, 0x9ffc000000d366f0) at deadlkres+0x700
fork_exit(0x9ffc000000d970a0, 0x0, 0xa000000000275550) at fork_exit+0x120
enter_userland() at enter_userland
KDB: enter: panic
# cat ddb.txt
db:0:kdb.enter.panic> run lockinfo
db:1:lockinfo> show locks
db:1:locks> show alllocks
Process 1125 (sh) thread 0xe0000000122e3200 (100077)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1123 (sh) thread 0xe000000012716d80 (100093)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1121 (sh) thread 0xe000000012719680 (100104)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1119 (sh) thread 0xe000000012716480 (100095)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1117 (sh) thread 0xe0000000122e2d80 (100078)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1113 (sh) thread 0xe00000001282e480 (100101)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1111 (sh) thread 0xe00000001282e000 (100102)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1109 (sh) thread 0xe0000000123e8480 (100073)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1107 (sshd) thread 0xe0000000123e8d80 (100071)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1106 (sh) thread 0xe0000000123e8900 (100072)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1104 (sh) thread 0xe000000012079b00 (100055)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124387b8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1102 (who) thread 0xe000000012719b00 (100103)
exclusive lockmgr ufs (ufs) r = 0 (0xe00000001261e2d8) locked @ /usr/src/sys/kern/vfs_vnops.c:373
Process 1100 (vi) thread 0xe0000000122e3680 (100076)
exclusive lockmgr bufwait (bufwait) r = 0 (0xa00000009e062028) locked @ /usr/src/sys/kern/vfs_bio.c:3081
exclusive lockmgr ufs (ufs) r = 0 (0xe0000000128dc2d8) locked @ /usr/src/sys/ufs/ffs/ffs_vfsops.c:1711
exclusive lockmgr ufs (ufs) r = 0 (0xe0000000124693e8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 22 (syncer) thread 0xe0000000122d8d80 (100064)
exclusive lockmgr ufs (ufs) r = 0 (0xe000000012633b38) locked @ /usr/src/sys/kern/vfs_subr.c:1720
db:1:alllocks> show lockedvnods
Locked vnodes
db:0:kdb.enter.panic> show pcpu
cpuid = 0
dynamic pcpu = 0x40040040ff1be380
curthread = 0xe000000011651200: pid 0 "deadlkres"
curpcb = 0
fpcurthread = none
idlethread = 0xe00000001117bb00: tid 100003 "idle: cpu0"
MD: vhpt = 0xe0000040ff000000
MD: lid = 0
MD: clock = 0x41c83545197
MD: clock_mode = 2
MD: clock_load = 0
MD: stats = 0x9ffc0000011c8330
MD: pmap = 0x9ffc000000e77240
spin locks held:
db:0:kdb.enter.panic> bt
Tracing pid 0 tid 100053 td 0xe000000011651200
kdb_enter(0x9ffc000000d40e50, 0x9ffc000000d40e50, 0x9ffc00000050b470, 0x40c) at kdb_enter+0x92
vpanic(0x9ffc000000d36760, 0xa000000000275518) at vpanic+0x2b0
panic(0x9ffc000000d36760, 0x9ffc000000d369b8, 0xe000000012079b00, 0x1b7ee8) at panic+0x80
deadlkres(0x9ffc000000d366f0, 0xe000000012079b00, 0x9ffc000000de9210, 0x9ffc000000d366f0) at deadlkres+0x700
fork_exit(0x9ffc000000d970a0, 0x0, 0xa000000000275550) at fork_exit+0x120
enter_userland() at enter_userland
db:0:kdb.enter.panic> ps
pid ppid pgrp uid state wmesg wchan cmd
1125 1124 1125 0 Ds ufs 0xe00000001261e2d8 sh
1124 935 935 0 S piperd 0xe00000001237ce70 cron
1123 1122 1123 2 Ds ufs 0xe00000001261e2d8 sh
1122 935 935 0 S piperd 0xe0000000127904d0 cron
1121 1120 1121 0 Ds ufs 0xe00000001261e2d8 sh
1120 935 935 0 S piperd 0xe000000012790000 cron
1119 1118 1119 0 Ds ufs 0xe00000001261e2d8 sh
1118 935 935 0 S piperd 0xe000000012820268 cron
1117 1116 1117 2 Ds ufs 0xe00000001261e2d8 sh
1116 935 935 0 S piperd 0xe000000012820738 cron
1113 1112 1113 0 Ds ufs 0xe00000001261e2d8 sh
1112 935 935 0 S piperd 0xe000000012820c08 cron
1111 1110 1111 0 Ds ufs 0xe00000001261e2d8 sh
1110 935 935 0 S piperd 0xe00000001237d340 cron
1109 1108 1109 2 Ds ufs 0xe00000001261e2d8 sh
1108 935 935 0 S piperd 0xe0000000128210d8 cron
1107 924 1107 0 Ds ufs 0xe00000001261e2d8 sshd
1106 1105 1106 0 Ds ufs 0xe00000001261e2d8 sh
1105 935 935 0 S piperd 0xe00000001237dce0 cron
1104 1103 1104 0 Ds ufs 0xe00000001261e2d8 sh
1103 935 935 0 S piperd 0xe000000012791810 cron
1102 1030 1102 0 D+ getblk 0xa00000009e0433c8 who
1100 1051 1051 1001 D+ biowr 0xa00000009e061fa0 vi
1051 1046 1051 1001 S+ wait 0xe000000012449720 sh
1046 1 1046 1001 Ss+ pause 0xe0000000124469e8 tcsh
1043 924 1043 0 Ds ufs 0xe0000000124387b8 sshd
1030 995 1030 0 S+ wait 0xe0000000122d7280 sh
995 1 995 0 Ss+ wait 0xe000000012796000 login
994 1 1 0 S ttydcd 0xe0000000116b90e8 getty
974 1 974 0 Ss select 0xe000000012301c40 inetd
935 1 935 0 Ss nanslp 0x9ffc000000e83eb8 cron
930 1 930 25 Ss pause 0xe0000000111797c8 sendmail
927 1 927 0 Ss select 0xe000000012300dc0 sendmail
924 1 924 0 Ss select 0xe0000000120aaa40 sshd
857 1 857 0 Ss select 0xe000000012300a40 ntpd
793 1 793 0 Ss select 0xe000000012300e40 casperd
792 1 24 0 S+ select 0xe000000012300940 casperd
763 1 763 0 Ds ufs 0xe000000012633b38 syslogd
638 1 638 0 Ss select 0xe0000000120aa2c0 devd
129 1 129 0 Ss pause 0xe0000000124480a8 adjkerntz
23 0 0 0 DL sdflush 0x9ffc000000e4fd50 [softdepflush]
22 0 0 0 DL getblk 0xa00000009e0422b8 [syncer]
21 0 0 0 DL vlruwt 0xe0000000122d1280 [vnlru]
20 0 0 0 DL psleep 0x9ffc000000e4f130 [bufdaemon]
19 0 0 0 DL pgzero 0x9ffc000000e500bc [pagezero]
18 0 0 0 DL psleep 0x9ffc000000e50024 [vmdaemon]
17 0 0 0 DL psleep 0x9ffc000000e50054 [pagedaemon]
7 0 0 0 DL waiting_ 0x9ffc0000011bc670 [sctp_iterator]
6 0 0 0 DL (threaded) [zfskern]
100052 D l2arc_fe 0x9ffc00000161b640 [l2arc_feed_thread]
100051 D arc_recl 0x9ffc00000160b820 [arc_reclaim_thread]
5 0 0 0 DL ispf 0xe0000000116a0000 [isp0: fc_thrd0]
4 0 0 0 DL idle 0xa000000000a26000 [mpt_recovery1]
3 0 0 0 DL idle 0xa000000000a0e000 [mpt_recovery0]
16 0 0 0 DL (threaded) [usb]
100039 D - 0xa000000000a0ae18 [usbus2]
100038 D - 0xa000000000a0adc0 [usbus2]
100037 D - 0xa000000000a0ad68 [usbus2]
100036 D - 0xa000000000a0ad10 [usbus2]
100034 D - 0xa000000000a01460 [usbus1]
100033 D - 0xa000000000a01408 [usbus1]
100032 D - 0xa000000000a013b0 [usbus1]
100031 D - 0xa000000000a01358 [usbus1]
100029 D - 0xa0000000009fd460 [usbus0]
100028 D - 0xa0000000009fd408 [usbus0]
100027 D - 0xa0000000009fd3b0 [usbus0]
100026 D - 0xa0000000009fd358 [usbus0]
15 0 0 0 DL tzpoll 0x9ffc000000e4dac0 [acpi_thermal]
2 0 0 0 DL (threaded) [cam]
100058 D ccb_scan 0x9ffc000000e503b8 [scanner]
100023 D - 0x9ffc000000e50500 [doneq0]
14 0 0 0 DL - 0x9ffc000000e4ddb0 [rand_harvestq]
13 0 0 0 DL (threaded) [geom]
100011 D - 0x9ffc000000e4e2f0 [g_down]
100010 D - 0x9ffc000000e4e2e8 [g_up]
100009 D - 0x9ffc000000e4e2d8 [g_event]
12 0 0 0 WL (threaded) [intr]
100048 I [swi0: uart uart++++]
100046 I [irq71: isp0]
100042 I [irq28: mpt1]
100040 I [irq27: mpt0]
100035 I [irq18: ehci0]
100030 I [irq17: ohci1]
100025 I [irq16: ohci0]
100019 I [swi6: task queue]
100018 I [swi6: Giant taskq]
100016 I [swi5: fast taskq]
100008 I [swi4: clock]
100007 I [swi4: clock]
100006 I [swi1: netisr 0]
100005 I [swi3: vm]
11 0 0 0 RL (threaded) [idle]
100004 Run CPU 1 [idle: cpu1]
100003 CanRun [idle: cpu0]
1 0 1 0 SLs wait 0xe0000000111784a0 [init]
10 0 0 0 DL audit_wo 0x9ffc0000011c0d08 [audit]
0 0 0 0 RLs (threaded) [kernel]
100057 D - 0xe00000001118c000 [CAM taskq]
100053 Run CPU 0 [deadlkres]
100050 D - 0xe000000011255300 [system_taskq_1]
100049 D - 0xe000000011255300 [system_taskq_0]
100045 D - 0xe000000011256900 [em1 taskq]
100044 D - 0xe000000011256b00 [em0 taskq]
100022 D - 0xe00000001118c100 [acpi_task_2]
100021 D - 0xe00000001118c100 [acpi_task_1]
100020 D - 0xe00000001118c100 [acpi_task_0]
100017 D - 0xe00000001118c600 [thread taskq]
100015 D - 0xe00000001118c900 [kqueue taskq]
100014 D - 0xe00000001118ca00 [ffs_trim taskq]
100012 D - 0xe00000001118d600 [firmware taskq]
100000 D swapin 0x9ffc000000e76808 [swapper]
1045 1043 1043 1001 Z sshd
db:0:kdb.enter.panic> alltrace
Tracing command sh pid 1125 tid 100077 td 0xe0000000122e3200
cpu_switch(0xe0000000122e3200, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000122e3200, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000122e3200, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000122e3578, 0xe0000000122e3200) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe0000000122e3200, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000335070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa000000000335050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe0000000122e3200, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003352e8, 0xa000000000335310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa000000000335310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003350f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa000000000335288, 0x200000, 0xe000000012438750, 0x0, 0xa000000000335318) at lookup+0xe30
namei(0xa000000000335288, 0xe000000012448940, 0x9ffc0000009b89f0, 0x0, 0xa0000000003352f8) at namei+0xb40
vn_open_cred(0xa000000000335288, 0xa000000000335390, 0x1c8, 0x0, 0xe000000011163600, 0xe00000001234b590) at vn_open_cred+0x500
vn_open(0xa000000000335288, 0xa000000000335390, 0x1c8, 0xe00000001234b590, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000122e3200, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe0000000122e3200, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe0000000122e3200, 0xa0000000003354e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe000000012448940, 0x100000, 0x9ffc000000e791d8, 0xe0000000122e3200, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1124 tid 100106 td 0xe000000012718d80
cpu_switch(0xe000000012718d80, 0xe00000001117b680, 0x9ffc000000ed5bd0, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012718d80, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012718d80, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001237ce70, 0x5c, 0xe0000000127190f8, 0xe000000012718d80) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001237ce70, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001237ce70, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe00000001237ce70, 0xe00000001237d0b0, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe000000012348484, 0xa00000000041d360, 0xa00000000041d378, 0x0, 0xe00000001237cf78) at pipe_read+0x740
dofileread(0xe000000012718d80, 0x6, 0xe000000012348460, 0xa00000000041d360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012718d80, 0x6, 0xa00000000041d360) at kern_readv+0xa0
sys_read(0xe000000012718d80, 0xa00000000041d4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe00000001293b720, 0x140c2c000, 0x2000, 0xe000000012718d80, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1123 tid 100093 td 0xe000000012716d80
cpu_switch(0xe000000012716d80, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012716d80, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012716d80, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000127170f8, 0xe000000012716d80) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe000000012716d80, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000003b5070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa0000000003b5050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe000000012716d80, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003b52e8, 0xa0000000003b5310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa0000000003b5310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003b50f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa0000000003b5288, 0x200000, 0xe000000012438750, 0x0, 0xa0000000003b5318) at lookup+0xe30
namei(0xa0000000003b5288, 0xe000000012449280, 0x9ffc0000009b89f0, 0x0, 0xa0000000003b52f8) at namei+0xb40
vn_open_cred(0xa0000000003b5288, 0xa0000000003b5390, 0x1c8, 0x0, 0xe000000011163700, 0xe00000001234b680) at vn_open_cred+0x500
vn_open(0xa0000000003b5288, 0xa0000000003b5390, 0x1c8, 0xe00000001234b680, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012716d80, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe000000012716d80, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe000000012716d80, 0xa0000000003b54e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe000000012449280, 0x100000, 0x9ffc000000e791d8, 0xe000000012716d80, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1122 tid 100070 td 0xe0000000116ab680
cpu_switch(0xe0000000116ab680, 0xe00000001117b680, 0x9ffc000000ed5f40, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000116ab680, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000116ab680, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe0000000127904d0, 0x5c, 0xe0000000116ab9f8, 0xe0000000116ab680) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000127904d0, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000127904d0, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe0000000127904d0, 0xe000000012790710, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe000000012348574, 0xa0000000002fd360, 0xa0000000002fd378, 0x0, 0xe0000000127905d8) at pipe_read+0x740
dofileread(0xe0000000116ab680, 0x6, 0xe000000012348550, 0xa0000000002fd360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000116ab680, 0x6, 0xa0000000002fd360) at kern_readv+0xa0
sys_read(0xe0000000116ab680, 0xa0000000002fd4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe0000000123cade0, 0x140c2c000, 0x2000, 0xe0000000116ab680, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1121 tid 100104 td 0xe000000012719680
cpu_switch(0xe000000012719680, 0xe00000001117b680, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012719680, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012719680, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000127199f8, 0xe000000012719680) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe000000012719680, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa00000000040d070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa00000000040d050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe000000012719680, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa00000000040d2e8, 0xa00000000040d310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa00000000040d310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa00000000040d0f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa00000000040d288, 0x200000, 0xe000000012438750, 0x0, 0xa00000000040d318) at lookup+0xe30
namei(0xa00000000040d288, 0xe0000000127964a0, 0x9ffc0000009b89f0, 0x0, 0xa00000000040d2f8) at namei+0xb40
vn_open_cred(0xa00000000040d288, 0xa00000000040d390, 0x1c8, 0x0, 0xe000000011165200, 0xe0000000123767d0) at vn_open_cred+0x500
vn_open(0xa00000000040d288, 0xa00000000040d390, 0x1c8, 0xe0000000123767d0, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012719680, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe000000012719680, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe000000012719680, 0xa00000000040d4e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe0000000127964a0, 0x100000, 0x9ffc000000e791d8, 0xe000000012719680, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1120 tid 100094 td 0xe000000012716900
cpu_switch(0xe000000012716900, 0xe00000001117bb00, 0x9ffc000000ed3e20, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012716900, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012716900, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe000000012790000, 0x5c, 0xe000000012716c78, 0xe000000012716900) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012790000, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012790000, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe000000012790000, 0xe000000012790240, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe00000001234b654, 0xa0000000003bd360, 0xa0000000003bd378, 0x0, 0xe000000012790108) at pipe_read+0x740
dofileread(0xe000000012716900, 0x6, 0xe00000001234b630, 0xa0000000003bd360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012716900, 0x6, 0xa0000000003bd360) at kern_readv+0xa0
sys_read(0xe000000012716900, 0xa0000000003bd4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe000000012448de0, 0x140c2c000, 0x2000, 0xe000000012716900, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1119 tid 100095 td 0xe000000012716480
cpu_switch(0xe000000012716480, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012716480, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012716480, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000127167f8, 0xe000000012716480) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe000000012716480, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000003c5070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa0000000003c5050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe000000012716480, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003c52e8, 0xa0000000003c5310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa0000000003c5310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003c50f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa0000000003c5288, 0x200000, 0xe000000012438750, 0x0, 0xa0000000003c5318) at lookup+0xe30
namei(0xa0000000003c5288, 0xe0000000127b0940, 0x9ffc0000009b89f0, 0x0, 0xa0000000003c52f8) at namei+0xb40
vn_open_cred(0xa0000000003c5288, 0xa0000000003c5390, 0x1c8, 0x0, 0xe000000011163900, 0xe000000012376960) at vn_open_cred+0x500
vn_open(0xa0000000003c5288, 0xa0000000003c5390, 0x1c8, 0xe000000012376960, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012716480, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe000000012716480, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe000000012716480, 0xa0000000003c54e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe0000000127b0940, 0x100000, 0x9ffc000000e791d8, 0xe000000012716480, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1118 tid 100097 td 0xe0000000122d8000
cpu_switch(0xe0000000122d8000, 0xe00000001117b680, 0x9ffc000000ed4eb0, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000122d8000, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000122d8000, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe000000012820268, 0x5c, 0xe0000000122d8378, 0xe0000000122d8000) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012820268, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012820268, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe000000012820268, 0xe0000000128204a8, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe000000012348894, 0xa0000000003d5360, 0xa0000000003d5378, 0x0, 0xe000000012820370) at pipe_read+0x740
dofileread(0xe0000000122d8000, 0x6, 0xe000000012348870, 0xa0000000003d5360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000122d8000, 0x6, 0xa0000000003d5360) at kern_readv+0xa0
sys_read(0xe0000000122d8000, 0xa0000000003d54e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe0000000122d0000, 0x140c2c000, 0x2000, 0xe0000000122d8000, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1117 tid 100078 td 0xe0000000122e2d80
cpu_switch(0xe0000000122e2d80, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000122e2d80, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000122e2d80, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000122e30f8, 0xe0000000122e2d80) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe0000000122e2d80, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa00000000033d070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa00000000033d050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe0000000122e2d80, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa00000000033d2e8, 0xa00000000033d310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa00000000033d310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa00000000033d0f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa00000000033d288, 0x200000, 0xe000000012438750, 0x0, 0xa00000000033d318) at lookup+0xe30
namei(0xa00000000033d288, 0xe0000000124484a0, 0x9ffc0000009b89f0, 0x0, 0xa00000000033d2f8) at namei+0xb40
vn_open_cred(0xa00000000033d288, 0xa00000000033d390, 0x1c8, 0x0, 0xe000000011163a00, 0xe000000012376c30) at vn_open_cred+0x500
vn_open(0xa00000000033d288, 0xa00000000033d390, 0x1c8, 0xe000000012376c30, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000122e2d80, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe0000000122e2d80, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe0000000122e2d80, 0xa00000000033d4e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe0000000124484a0, 0x100000, 0x9ffc000000e791d8, 0xe0000000122e2d80, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1116 tid 100090 td 0xe000000012717b00
cpu_switch(0xe000000012717b00, 0xe00000001117b680, 0x9ffc000000ed47f8, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012717b00, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012717b00, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe000000012820738, 0x5c, 0xe000000012717e78, 0xe000000012717b00) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012820738, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012820738, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe000000012820738, 0xe000000012820978, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123499c4, 0xa00000000039d360, 0xa00000000039d378, 0x0, 0xe000000012820840) at pipe_read+0x740
dofileread(0xe000000012717b00, 0x6, 0xe0000000123499a0, 0xa00000000039d360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012717b00, 0x6, 0xa00000000039d360) at kern_readv+0xa0
sys_read(0xe000000012717b00, 0xa00000000039d4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe0000000126c84a0, 0x140c2c000, 0x2000, 0xe000000012717b00, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1113 tid 100101 td 0xe00000001282e480
cpu_switch(0xe00000001282e480, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe00000001282e480, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001282e480, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe00000001282e7f8, 0xe00000001282e480) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe00000001282e480, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000003f5070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa0000000003f5050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe00000001282e480, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003f52e8, 0xa0000000003f5310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa0000000003f5310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003f50f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa0000000003f5288, 0x200000, 0xe000000012438750, 0x0, 0xa0000000003f5318) at lookup+0xe30
namei(0xa0000000003f5288, 0xe000000012797280, 0x9ffc0000009b89f0, 0x0, 0xa0000000003f52f8) at namei+0xb40
vn_open_cred(0xa0000000003f5288, 0xa0000000003f5390, 0x1c8, 0x0, 0xe000000011163b00, 0xe000000012348780) at vn_open_cred+0x500
vn_open(0xa0000000003f5288, 0xa0000000003f5390, 0x1c8, 0xe000000012348780, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe00000001282e480, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe00000001282e480, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe00000001282e480, 0xa0000000003f54e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe000000012797280, 0x100000, 0x9ffc000000e791d8, 0xe00000001282e480, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1112 tid 100084 td 0xe000000012636480
cpu_switch(0xe000000012636480, 0xe00000001117b680, 0x9ffc000000ed3ec0, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012636480, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012636480, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe000000012820c08, 0x5c, 0xe0000000126367f8, 0xe000000012636480) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012820c08, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012820c08, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe000000012820c08, 0xe000000012820e48, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123489d4, 0xa00000000036d360, 0xa00000000036d378, 0x0, 0xe000000012820d10) at pipe_read+0x740
dofileread(0xe000000012636480, 0x6, 0xe0000000123489b0, 0xa00000000036d360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012636480, 0x6, 0xa00000000036d360) at kern_readv+0xa0
sys_read(0xe000000012636480, 0xa00000000036d4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe0000000124464a0, 0x140c2c000, 0x2000, 0xe000000012636480, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1111 tid 100102 td 0xe00000001282e000
cpu_switch(0xe00000001282e000, 0xe00000001117b680, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe00000001282e000, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001282e000, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe00000001282e378, 0xe00000001282e000) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe00000001282e000, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000003fd070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa0000000003fd050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe00000001282e000, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003fd2e8, 0xa0000000003fd310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa0000000003fd310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003fd0f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa0000000003fd288, 0x200000, 0xe000000012438750, 0x0, 0xa0000000003fd318) at lookup+0xe30
namei(0xa0000000003fd288, 0xe000000012796de0, 0x9ffc0000009b89f0, 0x0, 0xa0000000003fd2f8) at namei+0xb40
vn_open_cred(0xa0000000003fd288, 0xa0000000003fd390, 0x1c8, 0x0, 0xe000000011164200, 0xe000000012348e10) at vn_open_cred+0x500
vn_open(0xa0000000003fd288, 0xa0000000003fd390, 0x1c8, 0xe000000012348e10, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe00000001282e000, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe00000001282e000, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe00000001282e000, 0xa0000000003fd4e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe000000012796de0, 0x100000, 0x9ffc000000e791d8, 0xe00000001282e000, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1110 tid 100096 td 0xe0000000122d8480
cpu_switch(0xe0000000122d8480, 0xe00000001117bb00, 0x9ffc000000ed5518, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000122d8480, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000122d8480, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001237d340, 0x5c, 0xe0000000122d87f8, 0xe0000000122d8480) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001237d340, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001237d340, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe00000001237d340, 0xe00000001237d580, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe00000001234b794, 0xa0000000003cd360, 0xa0000000003cd378, 0x0, 0xe00000001237d448) at pipe_read+0x740
dofileread(0xe0000000122d8480, 0x6, 0xe00000001234b770, 0xa0000000003cd360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000122d8480, 0x6, 0xa0000000003cd360) at kern_readv+0xa0
sys_read(0xe0000000122d8480, 0xa0000000003cd4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe0000000122d04a0, 0x140c2c000, 0x2000, 0xe0000000122d8480, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1109 tid 100073 td 0xe0000000123e8480
cpu_switch(0xe0000000123e8480, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000123e8480, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123e8480, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000123e87f8, 0xe0000000123e8480) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe0000000123e8480, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000315070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa000000000315050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe0000000123e8480, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003152e8, 0xa000000000315310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa000000000315310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003150f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa000000000315288, 0x200000, 0xe000000012438750, 0x0, 0xa000000000315318) at lookup+0xe30
namei(0xa000000000315288, 0xe0000000123ca000, 0x9ffc0000009b89f0, 0x0, 0xa0000000003152f8) at namei+0xb40
vn_open_cred(0xa000000000315288, 0xa000000000315390, 0x1c8, 0x0, 0xe000000011163d00, 0xe000000012349d60) at vn_open_cred+0x500
vn_open(0xa000000000315288, 0xa000000000315390, 0x1c8, 0xe000000012349d60, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123e8480, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe0000000123e8480, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe0000000123e8480, 0xa0000000003154e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe0000000123ca000, 0x100000, 0x9ffc000000e791d8, 0xe0000000123e8480, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1108 tid 100088 td 0xe0000000123e9200
cpu_switch(0xe0000000123e9200, 0xe00000001117b680, 0x9ffc000000ed5d60, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000123e9200, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123e9200, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe0000000128210d8, 0x5c, 0xe0000000123e9578, 0xe0000000123e9200) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000128210d8, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000128210d8, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe0000000128210d8, 0xe000000012821318, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe000000012348b14, 0xa00000000038d360, 0xa00000000038d378, 0x0, 0xe0000000128211e0) at pipe_read+0x740
dofileread(0xe0000000123e9200, 0x6, 0xe000000012348af0, 0xa00000000038d360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000123e9200, 0x6, 0xa00000000038d360) at kern_readv+0xa0
sys_read(0xe0000000123e9200, 0xa00000000038d4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe0000000126c8de0, 0x140c2c000, 0x2000, 0xe0000000123e9200, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sshd pid 1107 tid 100071 td 0xe0000000123e8d80
cpu_switch(0xe0000000123e8d80, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000123e8d80, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123e8d80, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000123e90f8, 0xe0000000123e8d80) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe0000000123e8d80, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000305070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa000000000305050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe0000000123e8d80, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000003052e8, 0xa000000000305310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa000000000305310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000003050f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa000000000305288, 0x200000, 0xe000000012438750, 0x0, 0xa000000000305318) at lookup+0xe30
namei(0xa000000000305288, 0xe0000000123ca940, 0x9ffc0000009b89f0, 0x0, 0xa0000000003052f8) at namei+0xb40
vn_open_cred(0xa000000000305288, 0xa000000000305390, 0x0, 0x0, 0xe000000011165000, 0xe0000000123768c0) at vn_open_cred+0x500
vn_open(0xa000000000305288, 0xa000000000305390, 0x0, 0xe0000000123768c0, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123e8d80, 0xffffffffffffff9c, 0x1400f99a0, 0x0, 0x0, 0x4010d000) at kern_openat+0x490
kern_open(0xe0000000123e8d80, 0x1400f99a0, 0x0, 0x100000, 0x4010d000) at kern_open+0x40
sys_open(0xe0000000123e8d80, 0xa0000000003054e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe0000000123ca940, 0x100000, 0x14010d000, 0xe0000000123e8d80, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1106 tid 100072 td 0xe0000000123e8900
cpu_switch(0xe0000000123e8900, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000123e8900, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123e8900, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe0000000123e8c78, 0xe0000000123e8900) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe0000000123e8900, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa00000000030d070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa00000000030d050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe0000000123e8900, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa00000000030d2e8, 0xa00000000030d310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa00000000030d310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa00000000030d0f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa00000000030d288, 0x200000, 0xe000000012438750, 0x0, 0xa00000000030d318) at lookup+0xe30
namei(0xa00000000030d288, 0xe0000000123ca4a0, 0x9ffc0000009b89f0, 0x0, 0xa00000000030d2f8) at namei+0xb40
vn_open_cred(0xa00000000030d288, 0xa00000000030d390, 0x1c8, 0x0, 0xe000000011163e00, 0xe000000012348a50) at vn_open_cred+0x500
vn_open(0xa00000000030d288, 0xa00000000030d390, 0x1c8, 0xe000000012348a50, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123e8900, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe0000000123e8900, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe0000000123e8900, 0xa00000000030d4e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe0000000123ca4a0, 0x100000, 0x9ffc000000e791d8, 0xe0000000123e8900, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1105 tid 100100 td 0xe00000001282e900
cpu_switch(0xe00000001282e900, 0xe00000001117b680, 0x9ffc000000ed4780, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe00000001282e900, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001282e900, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001237dce0, 0x5c, 0xe00000001282ec78, 0xe00000001282e900) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001237dce0, 0x5c, 0x9ffc000000d48810, 0x9ffc000000d48bd0) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001237dce0, 0x5c, 0x9ffc000000d4c260, 0x100, 0x9ffc00000051d880) at sleepq_wait_sig+0x30
_sleep(0xe00000001237dce0, 0xe00000001237df20, 0x5c, 0x9ffc000000d4c260, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe000000012348d44, 0xa0000000003ed360, 0xa0000000003ed378, 0x0, 0xe00000001237dde8) at pipe_read+0x740
dofileread(0xe00000001282e900, 0x6, 0xe000000012348d20, 0xa0000000003ed360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe00000001282e900, 0x6, 0xa0000000003ed360) at kern_readv+0xa0
sys_read(0xe00000001282e900, 0xa0000000003ed4e8, 0x9ffc000000a40c80, 0x48d) at sys_read+0x100
syscall(0xe000000012797720, 0x140c2c000, 0x2000, 0xe00000001282e900, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1104 tid 100055 td 0xe000000012079b00
cpu_switch(0xe000000012079b00, 0xe00000001117bb00, 0x9ffc000000ed4730, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe000000012079b00, 0xe00000001117bb00, 0x9ffc000000e9e118, 0x9ffc000000e9e100) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012079b00, 0x9ffc00000059de10) at mi_switch+0x3f0
sleepq_switch(0xe00000001261e2d8, 0x60, 0xe000000012079e78, 0xe000000012079b00) at sleepq_switch+0x2e0
sleepq_wait(0xe00000001261e2d8, 0x60, 0x9ffc000000d48810, 0xe000000012079b00, 0x9ffc0000004d4e20, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe00000001261e2d8, 0x200100, 0xe00000001261e3a0, 0x9ffc000000d34f08, 0x60, 0x0, 0x9ffc000000d560c8, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000285070, 0xe00000001261e2d8, 0x200100) at ffs_lock+0x160
VOP_LOCK1_APV(0x9ffc000000e13110, 0xa000000000285050, 0x9ffc000000df12f0) at VOP_LOCK1_APV+0x230
_vn_lock(0xe00000001261e270, 0x200100, 0x9ffc000000d560c8, 0x835) at _vn_lock+0x130
vget(0xe00000001261e270, 0x200100, 0xe000000012079b00, 0x9ffc0000011b9d48) at vget+0x100
cache_lookup(0xf, 0xa0000000002852e8, 0xa000000000285310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012438750, 0x0, 0xa000000000285310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000e13a50, 0xa0000000002850f8, 0x0, 0x9ffc000000d8a240) at VOP_LOOKUP_APV+0x270
lookup(0xa000000000285288, 0x200000, 0xe000000012438750, 0x0, 0xa000000000285318) at lookup+0xe30
namei(0xa000000000285288, 0xe000000011660000, 0x9ffc0000009b89f0, 0x0, 0xa0000000002852f8) at namei+0xb40
vn_open_cred(0xa000000000285288, 0xa000000000285390, 0x1c8, 0x0, 0xe000000011164000, 0xe000000012348960) at vn_open_cred+0x500
vn_open(0xa000000000285288, 0xa000000000285390, 0x1c8, 0xe000000012348960, 0x9ffc00000067ea60, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012079b00, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe791d8) at kern_openat+0x490
kern_open(0xe000000012079b00, 0x1400c39a0, 0x0, 0x100000, 0xe791d8) at kern_open+0x40
sys_open(0xe000000012079b00, 0xa0000000002854e8, 0x9ffc000000a40c80, 0x48d) at sys_open+0x40
syscall(0xe000000011660000, 0x100000, 0x9ffc000000e791d8, 0xe000000012079b00, 0x0, 0x0, 0x9ffc000000a3cca0, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1103 tid 100087 td 0xe0000000123e9680
cpu_switch(0xe0000000123e9680, 0xe00000001117b680, 0x9ffc000000ed3f60, 0x9ffc000000564d40) at cpu_switch+0xd0
sched_switch(0xe0000000123e9680, 0xe00000001117b680, 0x9ffc000000e9ee18, 0x9ffc000000e9ee00) at sched_switch+0x890
mi_switch(0x104, 0x
>How-To-Repeat:
>Fix:
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201403031103.s23B3TBV001131>