Date: Mon, 10 Mar 2014 13:00:01 GMT From: Anton Shterenlikht <mexas@bris.ac.uk> To: freebsd-ia64@FreeBSD.org Subject: ia64/187219: panic: 11.0-CURRENT #20 r260914: Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks Message-ID: <201403101300.s2AD01QQ080100@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
The following reply was made to PR ia64/187219; it has been noted by GNATS.
From: Anton Shterenlikht <mexas@bris.ac.uk>
To: bug-followup@freebsd.org
Cc:
Subject: ia64/187219: panic: 11.0-CURRENT #20 r260914: Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks
Date: Mon, 10 Mar 2014 05:29:53 -0700 (PDT)
Another one:
# cat info.last
Dump header from device /dev/da2p1
Architecture: ia64
Architecture Version: 1
Dump Length: 67584B (0 MB)
Blocksize: 512
Dumptime: Mon Mar 10 11:40:02 2014
Hostname: mech-as28.men.bris.ac.uk
Magic: FreeBSD Text Dump
Version String: FreeBSD 11.0-CURRENT #22 r260914: Wed Mar 5 10:12:04 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV
Panic String: deadlkres: possible deadlock detected for 0xe0000000123a6900, blocked for 1802278 ticks
Dump Parity: 2881296935
Bounds: 6
Dump Status: good
#
# cat panic.txt
deadlkres: possible deadlock detected for 0xe0000000123a6900, blocked for 1802278 ticks
#
# cat version.txt
FreeBSD 11.0-CURRENT #22 r260914: Wed Mar 5 10:12:04 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV
#
config.txt:
options CONFIG_AUTOGENERATED
ident ZEEV
machine ia64
cpu ITANIUM2
makeoptions MODULES_OVERRIDE=geom/geom_part opensolaris zfs acl_nfs4 fdescfs
makeoptions DEBUG=-g
options TEXTDUMP_VERBOSE
options TEXTDUMP_PREFERRED
options WITNESS_SKIPSPIN
options WITNESS
options INVARIANT_SUPPORT
options INVARIANTS
options DEADLKRES
options GDB
options DDB
options KDB_TRACE
options KDB
options _KPOSIX_PRIORITY_SCHEDULING
options QUOTA
options UFS_GJOURNAL
options UFS_DIRHASH
options UFS_ACL
options SYSVSHM
options SYSVSEM
options SYSVMSG
options STACK
options SOFTUPDATES
options SMP
options SCTP
options SCSI_DELAY=5000
options SCHED_ULE
options PSEUDOFS
options PROCFS
options PRINTF_BUFR_SIZE=128
options PREEMPTION
options P1003_1B_SEMAPHORES
options NFS_ROOT
options NFSD
options NFSLOCKD
options NFSCL
options MSDOSFS
options MD_ROOT
options MAC
options KTRACE
options INET6
options INET
options INCLUDE_CONFIG_FILE
options GEOM_LABEL
options FFS
options TMPFS
options NULLFS
options FDESCFS
options COMPAT_FREEBSD7
options CD9660
options CAPABILITIES
options CAPABILITY_MODE
options AUDIT
options NEW_PCIB
options GEOM_PART_MBR
options GEOM_PART_GPT
options GEOM_PART_BSD
device acpi
device io
device mem
device uart_ns8250
device miibus
device pci
device scbus
device usb
device isp
device ispfw
device mpt
device cd
device da
device pass
device sa
device ehci
device ohci
device uhci
device umass
device em
device ether
device faith
device firmware
device gif
device loop
device md
device puc
device random
device tun
device uart
device vlan
msgbuf.txt
GDB: debug ports: uart
GDB: current port: uart
KDB: debugger backends: ddb gdb
KDB: current backend: ddb
Copyright (c) 1992-2014 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 11.0-CURRENT #22 r260914: Wed Mar 5 10:12:04 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV ia64
gcc version 4.2.1 20070831 patched [FreeBSD]
WARNING: WITNESS option enabled, expect reduced performance.
CPU: Madison II (1600 MHz Itanium 2)
Origin = "GenuineIntel" Revision = 2
Features = 0x1<LB>
real memory = 10737418240 (10240 MB)
avail memory = 10508304384 (10021 MB)
FPSWA Revision = 0x10012, Entry = 0xe0000040ffcc4050
FreeBSD/SMP: Multiprocessor System Detected: 2 CPUs
cpu0: ACPI Id=0, SAPIC Id=0, SAPIC Eid=0 (BSP)
cpu1: ACPI Id=1, SAPIC Id=1, SAPIC Eid=0
random: <Software, Yarrow> initialized
Event timer "ITC" frequency 1600000000 Hz quality 1000
ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe0Block: 32/16 (20130823/tbfadt-603)
ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe1Block: 32/16 (20130823/tbfadt-603)
acpi0: <HP> on motherboard
acpi0: Power Button (fixed)
acpi0: Sleep Button (fixed)
cpu0: <ACPI CPU> on acpi0
cpu1: <ACPI CPU> on acpi0
Timecounter "ACPI-safe" frequency 3579545 Hz quality 850
acpi_timer0: <32-bit timer at 3.579545MHz> iomem 0xff5c1004-0xff5c1007 on acpi0
acpi_tz0: <Thermal Zone> on acpi0
pcib0: <ACPI Host-PCI bridge> on acpi0
pcib0: could not evaluate _ADR - AE_NOT_FOUND
pci0: <ACPI PCI bus> on pcib0
ohci0: <NEC uPD 9210 USB controller> mem 0x80002000-0x80002fff irq 16 at device 1.0 on pci0
usbus0 on ohci0
ohci1: <NEC uPD 9210 USB controller> mem 0x80001000-0x80001fff irq 17 at device 1.1 on pci0
usbus1 on ohci1
ehci0: <NEC uPD 720100 USB 2.0 controller> mem 0x80000000-0x800000ff irq 18 at device 1.2 on pci0
usbus2: EHCI version 0.95
usbus2 on ehci0
pci0: <mass storage, ATA> at device 2.0 (no driver attached)
pcib1: <ACPI Host-PCI bridge> on acpi0
pcib1: could not evaluate _ADR - AE_NOT_FOUND
pci32: <ACPI PCI bus> on pcib1
mpt0: <LSILogic 1030 Ultra4 Adapter> port 0x2100-0x21ff mem 0x903a0000-0x903bffff,0x90380000-0x9039ffff irq 27 at device 1.0 on pci32
mpt0: MPI Version=1.2.12.0
mpt1: <LSILogic 1030 Ultra4 Adapter> port 0x2000-0x20ff mem 0x90360000-0x9037ffff,0x90340000-0x9035ffff irq 28 at device 1.1 on pci32
mpt1: MPI Version=1.2.12.0
em0: <Intel(R) PRO/1000 Legacy Network Connection 1.0.6> port 0x2240-0x227f mem 0x90320000-0x9033ffff,0x90280000-0x902fffff irq 29 at device 2.0 on pci32
em0: Ethernet address: 00:13:21:5b:05:1c
em1: <Intel(R) PRO/1000 Legacy Network Connection 1.0.6> port 0x2200-0x223f mem 0x90300000-0x9031ffff irq 30 at device 2.1 on pci32
em1: Ethernet address: 00:13:21:5b:05:1d
pcib2: <ACPI Host-PCI bridge> on acpi0
pcib2: could not evaluate _ADR - AE_NOT_FOUND
pci64: <ACPI PCI bus> on pcib2
pcib3: <ACPI Host-PCI bridge> on acpi0
pcib3: could not evaluate _ADR - AE_NOT_FOUND
pci96: <ACPI PCI bus> on pcib3
pcib4: <ACPI Host-PCI bridge> on acpi0
pcib4: could not evaluate _ADR - AE_NOT_FOUND
pci128: <ACPI PCI bus> on pcib4
pcib5: <ACPI Host-PCI bridge> on acpi0
pcib5: could not evaluate _ADR - AE_NOT_FOUND
pci192: <ACPI PCI bus> on pcib5
isp0: <Qlogic ISP 2422 PCI FC-AL Adapter> port 0xc000-0xc0ff mem 0xe0040000-0xe0040fff irq 71 at device 1.0 on pci192
pcib6: <ACPI Host-PCI bridge> on acpi0
pcib6: could not evaluate _ADR - AE_NOT_FOUND
pci224: <ACPI PCI bus> on pcib6
uart0: <HP Auxiliary Diva Serial Port> mem 0xf8051000-0xf805100f irq 82 at device 1.0 on pci224
puc0: <HP Diva Serial [GSP] Multiport UART - Everest SP2> mem 0xf8050000-0xf8050fff,0xf8020000-0xf803ffff irq 82 at device 1.1 on pci224
uart1: <Non-standard ns8250 class UART with FIFOs> at port 1 on puc0
uart1: console (9600,n,8,1)
uart2: <Non-standard ns8250 class UART with FIFOs> at port 2 on puc0
uart3: <Non-standard ns8250 class UART with FIFOs> at port 3 on puc0
vgapci0: <VGA-compatible display> port 0xe000-0xe0ff mem 0xf0000000-0xf7ffffff,0xf8040000-0xf804ffff at device 2.0 on pci224
vgapci0: Boot video device
uart4: <16550 or compatible> iomem 0xff5e0000-0xff5e0007 irq 34 on acpi0
uart5: <16550 or compatible> iomem 0xff5e2000-0xff5e2007 irq 35 on acpi0
uart5: debug port (9600,n,8,1)
Timecounters tick every 1.000 msec
usbus0: 12Mbps Full Speed USB v1.0
usbus1: 12Mbps Full Speed USB v1.0
usbus2: 480Mbps High Speed USB v2.0
ugen1.1: <NEC> at usbus1
uhub0: <NEC OHCI root HUB, class 9/0, rev 1.00/1.00, addr 1> on usbus1
ugen0.1: <NEC> at usbus0
ugen2.1: <NEC> at usbus2
uhub1: <NEC OHCI root HUB, class 9/0, rev 1.00/1.00, addr 1> on usbus0
uhub2: <NEC EHCI root HUB, class 9/0, rev 2.00/1.00, addr 1> on usbus2
random: unblocking device.
uhub0: 2 ports with 2 removable, self powered
uhub1: 3 ports with 3 removable, self powered
uhub2: 5 ports with 5 removable, self powered
pass1 at isp0 bus 0 scbus2 target 0 lun 0
da1 at isp0 bus 0 scbus2 target 0 lun 1
pass1: <COMPAQ MSA1000 4.32> Fixed Storage Array SCSI-4 device
da1: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
pass1: Serial Number P56350GX3RX04Z
da1: Serial Number P56350GX3RX04Z
pass1: 200.000MB/s transfersda1: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 WWPN 0x500805f3000ec221 PortID 0x10000
PortID 0x10000
pass1: Command Queueing enabled
da1: Command Queueing enabled
da1: 69460MB (142255575 512 byte sectors: 255H 63S/T 8855C)
da2 at isp0 bus 0 scbus2 target 0 lun 2
da2: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da2: Serial Number P56350GX3RX04Z
da2: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da2: Command Queueing enabled
da2: 69460MB (142255575 512 byte sectors: 255H 63S/T 8855C)
da3 at isp0 bus 0 scbus2 target 0 lun 5
da3: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da3: Serial Number P56350GX3RX04Z
da3: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da3: Command Queueing enabled
da3: 140011MB (286744185 512 byte sectors: 255H 63S/T 17849C)
da0 at mpt0 bus 0 scbus0 target 0 lun 0
da0: <HP 73.4G ST373454LC HPC2> Fixed Direct Access SCSI-3 device
da0: Serial Number 3KP05D0W00007532H5SA
da0: 320.000MB/s transfers (160.000MHz, offset 63, 16bit)
da0: Command Queueing enabled
da0: 70007MB (143374738 512 byte sectors: 255H 63S/T 8924C)
WARNING: WITNESS option enabled, expect reduced performance.
Trying to mount root from ufs:/dev/da0p2 [rw]...
WARNING: / was not properly dismounted
<118>Setting hostuuid: ee652761-3dcc-11df-9074-0013215b051c.
<118>Setting hostid: 0x8968724b.
<118>Starting ddb.
<118>Entropy harvesting: interrupts ethernet point_to_point swi.
<118>Starting file system checks:
<118>/dev/da0p2: 563434 files, 3754911 used, 30861513 free (305153 frags, 3819545 blocks, 0.9% fragmentation)
<118>/dev/da1p1: FREE BLK COUNT(S) WRONG IN SUPERBLK (SALVAGED)
<118>/dev/da1p1: SUMMARY INFORMATION BAD (SALVAGED)
<118>/dev/da1p1: BLK(S) MISSING IN BIT MAPS (SALVAGED)
<118>/dev/da1p1: 2158 files, 81894 used, 17141467 free (443 frags, 2142628 blocks, 0.0% fragmentation)
<118>/dev/da3p1: 6085 files, 1328251 used, 68099252 free (6244 frags, 8511626 blocks, 0.0% fragmentation)
<118>Mounting local file systems:.
<118>Writing entropy file:.
<118>Setting hostname: mech-as28.men.bris.ac.uk.
<118>Starting Network: lo0 em0 em1.
<118>lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
<118> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
<118> inet6 ::1 prefixlen 128
<118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
<118> inet 127.0.0.1 netmask 0xff000000
<118> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
<118>em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC>
<118> ether 00:13:21:5b:05:1c
<118> inet 137.222.187.28 netmask 0xffffff00 broadcast 137.222.187.255
<118> inet6 fe80::213:21ff:fe5b:51c%em0 prefixlen 64 scopeid 0x1
<118> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
<118> media: Ethernet autoselect
<118> status: no carrier
<118>em1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC>
<118> ether 00:13:21:5b:05:1d
<118> inet 10.10.10.14 netmask 0xffffff00 broadcast 10.10.10.255
<118> inet6 fe80::213:21ff:fe5b:51d%em1 prefixlen 64 scopeid 0x2
<118> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
<118> media: Ethernet autoselect (100baseTX <full-duplex>)
<118> status: active
<118>Starting devd.
<118>Starting pflogd:
<118>add net default: gateway 137.222.187.250
<118>add net fe80::: gateway ::1
<118>add net ff02::: gateway ::1
<118>add net ::ffff:0.0.0.0: gateway ::1
<118>add net ::0.0.0.0: gateway ::1
<118>ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib
<118>Creating and/or trimming log files.
<118>Starting syslogd.
<118>savecore: reboot after panic: deadlkres: possible deadlock detected for 0xe00000001271e900, blocked for 1800893 ticks
<118>Mar 10 09:03:39 mech-as28 savecore: reboot after panic: deadlkres: possible deadlock detected for 0xe00000001271e900, blocked for 1800893 ticks
<118>savecore: writing core to /var/crash/textdump.tar.5
<118>/var/crash/vmcore.5 not found
<118>Starting casperd.
lock order reversal:
1st 0xa00000009e049220 bufwait (bufwait) @ /usr/src/sys/kern/vfs_bio.c:3081
2nd 0xe000000012130a00 dirhash (dirhash) @ /usr/src/sys/ufs/ufs/ufs_dirhash.c:284
KDB: stack backtrace:
db_trace_self(0x9ffc0000000d89c0) at db_trace_self+0x40
db_trace_self_wrapper(0x9ffc000000572870) at db_trace_self_wrapper+0x70
kdb_backtrace(0x9ffc000000e2b2d8, 0x9ffc0000005a8990) at kdb_backtrace+0xc0
_witness_debugger(0x1, 0x9ffc000000d2c138, 0x9ffc0000005ac370, 0xb9d, 0x9ffc000000d59270) at _witness_debugger+0x60
witness_checkorder(0xe000000012130a00, 0x9ffc000000d2ba80, 0x9ffc000000d59270, 0x11c, 0x0) at witness_checkorder+0x15c0
_sx_xlock(0xe000000012130a00, 0x0, 0x9ffc000000d59270, 0x11c) at _sx_xlock+0x120
ufsdirhash_acquire(0xe000000012423308, 0xe000000012130a00, 0x9ffc000000975d60) at ufsdirhash_acquire+0x50
ufsdirhash_add(0xe000000012423308, 0xa000000000279098, 0x934, 0xa000000000279040, 0x9ffc00000097dbf0, 0xda1) at ufsdirhash_add+0x20
ufs_direnter(0xe00000001267a1d8, 0xe000000012681ba8, 0xa000000000279098, 0xa000000000279318, 0xa00000009e049720, 0x0) at ufs_direnter+0x1260
ufs_mkdir(0xa000000000279370, 0xa000000000279098, 0xe00000001267a1d8) at ufs_mkdir+0x14f0
VOP_MKDIR_APV(0x9ffc000000df2660, 0xa000000000279370, 0xa000000000279318, 0x0, 0x9ffc0000006650c0, 0x818, 0x9ffc0000010015a0) at VOP_MKDIR_APV+0x220
kern_mkdirat(0xe0000000123a7b00, 0xffffffffffffff9c, 0x140804040, 0x0, 0x1c0) at kern_mkdirat+0x3a0
kern_mkdir(0xe0000000123a7b00, 0x140804040, 0x0, 0x1c0) at kern_mkdir+0x40
sys_mkdir(0xe0000000123a7b00, 0xa0000000002794e8, 0x9ffc000000a2cc80, 0x48d) at sys_mkdir+0x30
syscall(0xe00000001114ede0, 0x1c0, 0x9ffc000000a29630, 0xe0000000123a7b00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
<118>Clearing /tmp (X related).
<118>Recovering vi editor sessions:.
<118>Updating motd:.
<118>Mounting late file systems:.
<118>Starting ntpd.
<118>Starting bsdstats.
<118>Posting monthly OS statistics to rpt.bsdstats.org
<118>Performing sanity check on sshd configuration.
<118>Starting sshd.
<118>Starting sendmail_submit.
<118>Starting sendmail_msp_queue.
<118>Starting cron.
<118>Local package initialization:cd: /var/portbuild/: No such file or directory
<118>.
<118>Starting inetd.
<118>Starting background file system checks in 60 seconds.
<118>
<118>Mon Mar 10 09:03:49 GMT 2014
<118>Mar 10 11:08:30 mech-as28 su: mexas to root on /dev/pts/0
panic: deadlkres: possible deadlock detected for 0xe0000000123a6900, blocked for 1802278 ticks
cpuid = 0
KDB: stack backtrace:
db_trace_self(0x9ffc0000000d89c0) at db_trace_self+0x40
db_trace_self_wrapper(0x9ffc000000572870) at db_trace_self_wrapper+0x70
kdb_backtrace(0x9ffc000000e2b2d8, 0x9ffc0000004f99e0, 0x40c, 0x9ffc0000010015a0) at kdb_backtrace+0xc0
vpanic(0x9ffc000000d16c10, 0xa00000009dfff518) at vpanic+0x260
panic(0x9ffc000000d16c10, 0x9ffc000000d16e68, 0xe0000000123a6900, 0x1b8026) at panic+0x80
deadlkres(0x9ffc000000d16ba0, 0xe0000000123a6900, 0x9ffc000000dc8358, 0x9ffc000000d16ba0) at deadlkres+0x700
fork_exit(0x9ffc000000d6b130, 0x0, 0xa00000009dfff550) at fork_exit+0x120
enter_userland() at enter_userland
KDB: enter: panic
ddb.txt
db:0:kdb.enter.panic> run lockinfo
db:1:lockinfo> show locks
db:1:locks> show alllocks
Process 1281 (sh) thread 0xe0000000123a7680 (100065)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1279 (sshd) thread 0xe000000012a58900 (100100)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1278 (sh) thread 0xe000000012a58480 (100101)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1274 (sh) thread 0xe0000000123adb00 (100075)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1272 (sh) thread 0xe000000011781b00 (100089)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1270 (sshd) thread 0xe0000000123a6000 (100070)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1269 (sh) thread 0xe000000012799680 (100093)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1267 (sh) thread 0xe000000012799b00 (100092)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1265 (sh) thread 0xe0000000123a7200 (100066)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1263 (sh) thread 0xe000000012663680 (100080)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1261 (sh) thread 0xe000000012662d80 (100082)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1259 (sh) thread 0xe0000000123a6900 (100068)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1257 (sh) thread 0xe000000012433680 (100087)
exclusive lockmgr ufs (ufs) r = 0 (0xe000000012899688) locked @ /usr/src/sys/kern/vfs_vnops.c:373
Process 1243 (pkg) thread 0xe0000000123a6d80 (100067)
exclusive lockmgr ufs (ufs) r = 0 (0xe00000001286f100) locked @ /usr/src/sys/kern/vfs_vnops.c:373
Process 1002 (login) thread 0xe000000012662900 (100083)
shared lockmgr ufs (ufs) r = 0 (0xe0000000124474b0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 20 (syncer) thread 0xe000000012344900 (100059)
exclusive lockmgr ufs (ufs) r = 0 (0xe000000012680068) locked @ /usr/src/sys/kern/vfs_subr.c:1720
db:1:alllocks> show lockedvnods
Locked vnodes
db:0:kdb.enter.panic> show pcpu
cpuid = 0
dynamic pcpu = 0x40040040ff1df980
curthread = 0xe000000011780480: pid 0 "deadlkres"
curpcb = 0
fpcurthread = none
idlethread = 0xe000000011151b00: tid 100003 "idle: cpu0"
MD: vhpt = 0xe0000040ff000000
MD: lid = 0
MD: clock = 0xde8220a6b52
MD: clock_mode = 2
MD: clock_load = 0
MD: stats = 0x9ffc0000011a4a30
MD: pmap = 0x9ffc000000e53920
spin locks held:
db:0:kdb.enter.panic> bt
Tracing pid 0 tid 100049 td 0xe000000011780480
kdb_enter(0x9ffc000000d21190, 0x9ffc000000d21190, 0x9ffc0000004f9a30, 0x40c) at kdb_enter+0x92
vpanic(0x9ffc000000d16c10, 0xa00000009dfff518) at vpanic+0x2b0
panic(0x9ffc000000d16c10, 0x9ffc000000d16e68, 0xe0000000123a6900, 0x1b8026) at panic+0x80
deadlkres(0x9ffc000000d16ba0, 0xe0000000123a6900, 0x9ffc000000dc8358, 0x9ffc000000d16ba0) at deadlkres+0x700
fork_exit(0x9ffc000000d6b130, 0x0, 0xa00000009dfff550) at fork_exit+0x120
enter_userland() at enter_userland
db:0:kdb.enter.panic> ps
pid ppid pgrp uid state wmesg wchan cmd
1281 1280 1281 0 Ds ufs 0xe000000012899688 sh
1280 942 942 0 S piperd 0xe000000012a7ba78 cron
1279 931 1279 0 Ds ufs 0xe000000012899688 sshd
1278 1277 1278 0 Ds ufs 0xe000000012899688 sh
1277 942 942 0 S piperd 0xe000000012794e70 cron
1274 1273 1274 2 Ds ufs 0xe000000012899688 sh
1273 942 942 0 S piperd 0xe0000000123dd0d8 cron
1272 1271 1272 0 Ds ufs 0xe000000012899688 sh
1271 942 942 0 S piperd 0xe000000012795a78 cron
1270 931 1270 0 Ds ufs 0xe000000012899688 sshd
1269 1268 1269 0 Ds ufs 0xe000000012899688 sh
1268 942 942 0 S piperd 0xe0000000123dc268 cron
1267 1266 1267 2 Ds ufs 0xe000000012899688 sh
1266 942 942 0 S piperd 0xe0000000123ddce0 cron
1265 1264 1265 0 Ds ufs 0xe000000012899688 sh
1264 942 942 0 S piperd 0xe0000000123dce70 cron
1263 1262 1263 0 Ds ufs 0xe000000012899688 sh
1262 942 942 0 S piperd 0xe0000000127955a8 cron
1261 1260 1261 2 Ds ufs 0xe000000012899688 sh
1260 942 942 0 S piperd 0xe0000000123dda78 cron
1259 1258 1259 0 Ds ufs 0xe000000012899688 sh
1258 942 942 0 S piperd 0xe000000012794738 cron
1257 1243 1243 0 D+ getblk 0xa00000009e032e48 sh
1243 1241 1243 0 DE+ getblk 0xa00000009e032510 pkg
1241 1240 1241 0 S+ wait 0xe000000012340000 sh
1240 1236 1240 1001 S+ wait 0xe000000012429720 su
1236 1 1236 1001 Ss+ pause 0xe0000000123437c8 tcsh
1233 931 1233 0 Ds ufs 0xe0000000124474b0 sshd
1002 1 1002 0 Ds+ ufs 0xe000000012899688 login
1001 1 1 0 S ttydcd 0xe00000001173b0e8 getty
981 1 981 0 Ss select 0xe00000001214c7c0 inetd
942 1 942 0 Ss nanslp 0x9ffc000000e60598 cron
937 1 937 25 Ss pause 0xe00000001114f328 sendmail
934 1 934 0 Ss select 0xe00000001178b6c0 sendmail
931 1 931 0 Ss select 0xe00000001178ba40 sshd
864 1 864 0 Ss select 0xe0000000122f3cc0 ntpd
800 1 800 0 Ss select 0xe00000001178bb40 casperd
799 1 23 0 S+ select 0xe0000000117372c0 casperd
770 1 770 0 Ds ufs 0xe000000012680068 syslogd
645 1 645 0 Ss select 0xe0000000122159c0 devd
136 1 136 0 Ss pause 0xe000000012437328 adjkerntz
22 0 0 0 DL sdflush 0x9ffc000000e2c480 [softdepflush]
21 0 0 0 DL vlruwt 0xe000000012340de0 [vnlru]
20 0 0 0 DL getblk 0xa00000009e032160 [syncer]
19 0 0 0 DL psleep 0x9ffc000000e2b860 [bufdaemon]
18 0 0 0 DL pgzero 0x9ffc000000e2c7ec [pagezero]
17 0 0 0 DL psleep 0x9ffc000000e2c754 [vmdaemon]
9 0 0 0 DL psleep 0x9ffc000000e2c784 [pagedaemon]
6 0 0 0 DL waiting_ 0x9ffc000001198d70 [sctp_iterator]
5 0 0 0 DL ispf 0xe00000001173c800 [isp0: fc_thrd0]
4 0 0 0 DL idle 0xa000000000a22000 [mpt_recovery1]
3 0 0 0 DL idle 0xa000000000a0a000 [mpt_recovery0]
16 0 0 0 DL (threaded) [usb]
100039 D - 0xa000000000a06e18 [usbus2]
100038 D - 0xa000000000a06dc0 [usbus2]
100037 D - 0xa000000000a06d68 [usbus2]
100036 D - 0xa000000000a06d10 [usbus2]
100034 D - 0xa0000000009fd460 [usbus1]
100033 D - 0xa0000000009fd408 [usbus1]
100032 D - 0xa0000000009fd3b0 [usbus1]
100031 D - 0xa0000000009fd358 [usbus1]
100029 D - 0xa0000000009f9460 [usbus0]
100028 D - 0xa0000000009f9408 [usbus0]
100027 D - 0xa0000000009f93b0 [usbus0]
100026 D - 0xa0000000009f9358 [usbus0]
15 0 0 0 DL tzpoll 0x9ffc000000e2a288 [acpi_thermal]
2 0 0 0 DL (threaded) [cam]
100054 D ccb_scan 0x9ffc000000e2cb38 [scanner]
100015 D - 0x9ffc000000e2cc80 [doneq0]
14 0 0 0 DL - 0x9ffc000000e2a578 [rand_harvestq]
13 0 0 0 DL (threaded) [geom]
100011 D - 0x9ffc000000e2aa28 [g_down]
100010 D - 0x9ffc000000e2aa20 [g_up]
100009 D - 0x9ffc000000e2aa10 [g_event]
12 0 0 0 WL (threaded) [intr]
100048 I [swi0: uart uart++++]
100046 I [irq71: isp0]
100042 I [irq28: mpt1]
100040 I [irq27: mpt0]
100035 I [irq18: ehci0]
100030 I [irq17: ohci1]
100025 I [irq16: ohci0]
100023 I [swi6: Giant taskq]
100021 I [swi5: fast taskq]
100014 I [swi6: task queue]
100008 I [swi1: netisr 0]
100007 I [swi3: vm]
100006 I [swi4: clock]
100005 I [swi4: clock]
11 0 0 0 RL (threaded) [idle]
100004 Run CPU 1 [idle: cpu1]
100003 CanRun [idle: cpu0]
1 0 1 0 SLs wait 0xe00000001114e4a0 [init]
10 0 0 0 DL audit_wo 0x9ffc00000119d408 [audit]
0 0 0 0 RLs (threaded) [kernel]
100053 D - 0xe000000011142300 [CAM taskq]
100049 Run CPU 0 [deadlkres]
100045 D - 0xe000000011207e00 [em1 taskq]
100044 D - 0xe000000011208200 [em0 taskq]
100022 D - 0xe000000011141a00 [thread taskq]
100020 D - 0xe000000011141d00 [kqueue taskq]
100019 D - 0xe000000011141e00 [ffs_trim taskq]
100018 D - 0xe000000011142000 [acpi_task_2]
100017 D - 0xe000000011142000 [acpi_task_1]
100016 D - 0xe000000011142000 [acpi_task_0]
100012 D - 0xe000000011143100 [firmware taskq]
100000 D swapin 0x9ffc000000e52ee8 [swapper]
1235 1233 1233 1001 Z sshd
db:0:kdb.enter.panic> alltrace
Tracing command sh pid 1281 tid 100065 td 0xe0000000123a7680
cpu_switch(0xe0000000123a7680, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123a7680, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123a7680, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000123a79f8, 0xe0000000123a7680) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe0000000123a7680, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000281070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000281050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe0000000123a7680, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000002812e8, 0xa000000000281310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000281310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002810f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000281288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000281318) at lookup+0xe10
namei(0xa000000000281288, 0xe0000000123e5720, 0x9ffc0000009a5410, 0x0, 0xa0000000002812f8) at namei+0xb40
vn_open_cred(0xa000000000281288, 0xa000000000281390, 0x8a8, 0x0, 0xe00000001229d200, 0xe0000000123b3720) at vn_open_cred+0x500
vn_open(0xa000000000281288, 0xa000000000281390, 0x8a8, 0xe0000000123b3720, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123a7680, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe0000000123a7680, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe0000000123a7680, 0xa0000000002814e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe0000000123e5720, 0x100000, 0x9ffc000000e558b8, 0xe0000000123a7680, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1280 tid 100103 td 0xe00000001279bb00
cpu_switch(0xe00000001279bb00, 0xe000000011151680, 0x9ffc000000eb2370, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe00000001279bb00, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001279bb00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012a7ba78, 0x5c, 0xe00000001279be78, 0xe00000001279bb00) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012a7ba78, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012a7ba78, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe000000012a7ba78, 0xe000000012a7bcb8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b50b4, 0xa0000000003b1360, 0xa0000000003b1378, 0x0, 0xe000000012a7bb80) at pipe_read+0x740
dofileread(0xe00000001279bb00, 0x6, 0xe0000000123b5090, 0xa0000000003b1360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe00000001279bb00, 0x6, 0xa0000000003b1360) at kern_readv+0xa0
sys_read(0xe00000001279bb00, 0xa0000000003b14e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000123b1720, 0x140c26000, 0x2000, 0xe00000001279bb00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sshd pid 1279 tid 100100 td 0xe000000012a58900
cpu_switch(0xe000000012a58900, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012a58900, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012a58900, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe000000012a58c78, 0xe000000012a58900) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000012a58900, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000399070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000399050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000012a58900, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000003992e8, 0xa000000000399310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000399310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000003990f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000399288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000399318) at lookup+0xe10
namei(0xa000000000399288, 0xe000000012a06940, 0x9ffc0000009a5410, 0x0, 0xa0000000003992f8) at namei+0xb40
vn_open_cred(0xa000000000399288, 0xa000000000399390, 0x0, 0x0, 0xe000000011143200, 0xe0000000123b5130) at vn_open_cred+0x500
vn_open(0xa000000000399288, 0xa000000000399390, 0x0, 0xe0000000123b5130, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012a58900, 0xffffffffffffff9c, 0x1400f99a0, 0x0, 0x0, 0x4010d000) at kern_openat+0x490
kern_open(0xe000000012a58900, 0x1400f99a0, 0x0, 0x100000, 0x4010d000) at kern_open+0x40
sys_open(0xe000000012a58900, 0xa0000000003994e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe000000012a06940, 0x100000, 0x14010d000, 0xe000000012a58900, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1278 tid 100101 td 0xe000000012a58480
cpu_switch(0xe000000012a58480, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012a58480, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012a58480, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe000000012a587f8, 0xe000000012a58480) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000012a58480, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000003a1070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000003a1050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000012a58480, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000003a12e8, 0xa0000000003a1310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa0000000003a1310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000003a10f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000003a1288, 0x200000, 0xe000000012447448, 0x0, 0xa0000000003a1318) at lookup+0xe10
namei(0xa0000000003a1288, 0xe000000012a064a0, 0x9ffc0000009a5410, 0x0, 0xa0000000003a12f8) at namei+0xb40
vn_open_cred(0xa0000000003a1288, 0xa0000000003a1390, 0x8a8, 0x0, 0xe00000001230c000, 0xe0000000123b2a50) at vn_open_cred+0x500
vn_open(0xa0000000003a1288, 0xa0000000003a1390, 0x8a8, 0xe0000000123b2a50, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012a58480, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe000000012a58480, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe000000012a58480, 0xa0000000003a14e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe000000012a064a0, 0x100000, 0x9ffc000000e558b8, 0xe000000012a58480, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1277 tid 100102 td 0xe000000012a58000
cpu_switch(0xe000000012a58000, 0xe000000011151680, 0x9ffc000000eb0ed0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012a58000, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012a58000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012794e70, 0x5c, 0xe000000012a58378, 0xe000000012a58000) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012794e70, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012794e70, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe000000012794e70, 0xe0000000127950b0, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b4e34, 0xa0000000003a9360, 0xa0000000003a9378, 0x0, 0xe000000012794f78) at pipe_read+0x740
dofileread(0xe000000012a58000, 0x6, 0xe0000000123b4e10, 0xa0000000003a9360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012a58000, 0x6, 0xa0000000003a9360) at kern_readv+0xa0
sys_read(0xe000000012a58000, 0xa0000000003a94e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a06000, 0x140c26000, 0x2000, 0xe000000012a58000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1274 tid 100075 td 0xe0000000123adb00
cpu_switch(0xe0000000123adb00, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123adb00, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123adb00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000123ade78, 0xe0000000123adb00) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe0000000123adb00, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000002d1070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000002d1050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe0000000123adb00, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000002d12e8, 0xa0000000002d1310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa0000000002d1310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002d10f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000002d1288, 0x200000, 0xe000000012447448, 0x0, 0xa0000000002d1318) at lookup+0xe10
namei(0xa0000000002d1288, 0xe0000000124284a0, 0x9ffc0000009a5410, 0x0, 0xa0000000002d12f8) at namei+0xb40
vn_open_cred(0xa0000000002d1288, 0xa0000000002d1390, 0x8a8, 0x0, 0xe00000001215db00, 0xe0000000123d6dc0) at vn_open_cred+0x500
vn_open(0xa0000000002d1288, 0xa0000000002d1390, 0x8a8, 0xe0000000123d6dc0, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123adb00, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe0000000123adb00, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe0000000123adb00, 0xa0000000002d14e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe0000000124284a0, 0x100000, 0x9ffc000000e558b8, 0xe0000000123adb00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1273 tid 100099 td 0xe000000012663b00
cpu_switch(0xe000000012663b00, 0xe000000011151680, 0x9ffc000000eb0660, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012663b00, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012663b00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123dd0d8, 0x5c, 0xe000000012663e78, 0xe000000012663b00) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123dd0d8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123dd0d8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123dd0d8, 0xe0000000123dd318, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b2074, 0xa000000000391360, 0xa000000000391378, 0x0, 0xe0000000123dd1e0) at pipe_read+0x740
dofileread(0xe000000012663b00, 0x6, 0xe0000000123b2050, 0xa000000000391360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012663b00, 0x6, 0xa000000000391360) at kern_readv+0xa0
sys_read(0xe000000012663b00, 0xa0000000003914e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a06de0, 0x140c26000, 0x2000, 0xe000000012663b00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1272 tid 100089 td 0xe000000011781b00
cpu_switch(0xe000000011781b00, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000011781b00, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000011781b00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe000000011781e78, 0xe000000011781b00) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000011781b00, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000341070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000341050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000011781b00, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000003412e8, 0xa000000000341310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000341310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000003410f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000341288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000341318) at lookup+0xe10
namei(0xa000000000341288, 0xe0000000123404a0, 0x9ffc0000009a5410, 0x0, 0xa0000000003412f8) at namei+0xb40
vn_open_cred(0xa000000000341288, 0xa000000000341390, 0x8a8, 0x0, 0xe0000000121c8500, 0xe0000000123b2aa0) at vn_open_cred+0x500
vn_open(0xa000000000341288, 0xa000000000341390, 0x8a8, 0xe0000000123b2aa0, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000011781b00, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe000000011781b00, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe000000011781b00, 0xa0000000003414e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe0000000123404a0, 0x100000, 0x9ffc000000e558b8, 0xe000000011781b00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1271 tid 100098 td 0xe000000012798000
cpu_switch(0xe000000012798000, 0xe000000011151680, 0x9ffc000000eb0a70, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012798000, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012798000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012795a78, 0x5c, 0xe000000012798378, 0xe000000012798000) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe000000012795a78, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe000000012795a78, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe000000012795a78, 0xe000000012795cb8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b2164, 0xa000000000389360, 0xa000000000389378, 0x0, 0xe000000012795b80) at pipe_read+0x740
dofileread(0xe000000012798000, 0x6, 0xe0000000123b2140, 0xa000000000389360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012798000, 0x6, 0xa000000000389360) at kern_readv+0xa0
sys_read(0xe000000012798000, 0xa0000000003894e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a07280, 0x140c26000, 0x2000, 0xe000000012798000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sshd pid 1270 tid 100070 td 0xe0000000123a6000
cpu_switch(0xe0000000123a6000, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123a6000, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123a6000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000123a6378, 0xe0000000123a6000) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe0000000123a6000, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000002a9070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000002a9050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe0000000123a6000, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000002a92e8, 0xa0000000002a9310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa0000000002a9310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002a90f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000002a9288, 0x200000, 0xe000000012447448, 0x0, 0xa0000000002a9318) at lookup+0xe10
namei(0xa0000000002a9288, 0xe0000000123e4000, 0x9ffc0000009a5410, 0x0, 0xa0000000002a92f8) at namei+0xb40
vn_open_cred(0xa0000000002a9288, 0xa0000000002a9390, 0x0, 0x0, 0xe000000011143200, 0xe0000000123b2cd0) at vn_open_cred+0x500
vn_open(0xa0000000002a9288, 0xa0000000002a9390, 0x0, 0xe0000000123b2cd0, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123a6000, 0xffffffffffffff9c, 0x1400f99a0, 0x0, 0x0, 0x4010d000) at kern_openat+0x490
kern_open(0xe0000000123a6000, 0x1400f99a0, 0x0, 0x100000, 0x4010d000) at kern_open+0x40
sys_open(0xe0000000123a6000, 0xa0000000002a94e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe0000000123e4000, 0x100000, 0x14010d000, 0xe0000000123a6000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1269 tid 100093 td 0xe000000012799680
cpu_switch(0xe000000012799680, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012799680, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012799680, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000127999f8, 0xe000000012799680) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000012799680, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000361070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000361050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000012799680, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000003612e8, 0xa000000000361310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000361310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000003610f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000361288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000361318) at lookup+0xe10
namei(0xa000000000361288, 0xe000000012428de0, 0x9ffc0000009a5410, 0x0, 0xa0000000003612f8) at namei+0xb40
vn_open_cred(0xa000000000361288, 0xa000000000361390, 0x8a8, 0x0, 0xe00000001230d400, 0xe0000000123b2fa0) at vn_open_cred+0x500
vn_open(0xa000000000361288, 0xa000000000361390, 0x8a8, 0xe0000000123b2fa0, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012799680, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe000000012799680, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe000000012799680, 0xa0000000003614e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe000000012428de0, 0x100000, 0x9ffc000000e558b8, 0xe000000012799680, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1268 tid 100097 td 0xe000000012798480
cpu_switch(0xe000000012798480, 0xe000000011151680, 0x9ffc000000eb1fb0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012798480, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012798480, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123dc268, 0x5c, 0xe0000000127987f8, 0xe000000012798480) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123dc268, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123dc268, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123dc268, 0xe0000000123dc4a8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b22f4, 0xa000000000381360, 0xa000000000381378, 0x0, 0xe0000000123dc370) at pipe_read+0x740
dofileread(0xe000000012798480, 0x6, 0xe0000000123b22d0, 0xa000000000381360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012798480, 0x6, 0xa000000000381360) at kern_readv+0xa0
sys_read(0xe000000012798480, 0xa0000000003814e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a07720, 0x140c26000, 0x2000, 0xe000000012798480, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1267 tid 100092 td 0xe000000012799b00
cpu_switch(0xe000000012799b00, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012799b00, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012799b00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe000000012799e78, 0xe000000012799b00) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000012799b00, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000359070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000359050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000012799b00, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000003592e8, 0xa000000000359310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000359310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000003590f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000359288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000359318) at lookup+0xe10
namei(0xa000000000359288, 0xe000000012429280, 0x9ffc0000009a5410, 0x0, 0xa0000000003592f8) at namei+0xb40
vn_open_cred(0xa000000000359288, 0xa000000000359390, 0x8a8, 0x0, 0xe00000001229e600, 0xe0000000123b2910) at vn_open_cred+0x500
vn_open(0xa000000000359288, 0xa000000000359390, 0x8a8, 0xe0000000123b2910, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012799b00, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe000000012799b00, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe000000012799b00, 0xa0000000003594e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe000000012429280, 0x100000, 0x9ffc000000e558b8, 0xe000000012799b00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1266 tid 100096 td 0xe000000012798900
cpu_switch(0xe000000012798900, 0xe000000011151680, 0x9ffc000000eb0e80, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012798900, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012798900, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123ddce0, 0x5c, 0xe000000012798c78, 0xe000000012798900) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123ddce0, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123ddce0, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123ddce0, 0xe0000000123ddf20, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b2344, 0xa000000000379360, 0xa000000000379378, 0x0, 0xe0000000123ddde8) at pipe_read+0x740
dofileread(0xe000000012798900, 0x6, 0xe0000000123b2320, 0xa000000000379360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012798900, 0x6, 0xa000000000379360) at kern_readv+0xa0
sys_read(0xe000000012798900, 0xa0000000003794e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a0c000, 0x140c26000, 0x2000, 0xe000000012798900, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1265 tid 100066 td 0xe0000000123a7200
cpu_switch(0xe0000000123a7200, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123a7200, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123a7200, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000123a7578, 0xe0000000123a7200) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe0000000123a7200, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000289070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000289050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe0000000123a7200, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000002892e8, 0xa000000000289310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000289310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002890f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000289288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000289318) at lookup+0xe10
namei(0xa000000000289288, 0xe0000000123e5280, 0x9ffc0000009a5410, 0x0, 0xa0000000002892f8) at namei+0xb40
vn_open_cred(0xa000000000289288, 0xa000000000289390, 0x8a8, 0x0, 0xe00000001229e300, 0xe0000000123b2b90) at vn_open_cred+0x500
vn_open(0xa000000000289288, 0xa000000000289390, 0x8a8, 0xe0000000123b2b90, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe0000000123a7200, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe0000000123a7200, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe0000000123a7200, 0xa0000000002894e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe0000000123e5280, 0x100000, 0x9ffc000000e558b8, 0xe0000000123a7200, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1264 tid 100095 td 0xe000000012798d80
cpu_switch(0xe000000012798d80, 0xe000000011151680, 0x9ffc000000eb22d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012798d80, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012798d80, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123dce70, 0x5c, 0xe0000000127990f8, 0xe000000012798d80) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123dce70, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123dce70, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123dce70, 0xe0000000123dd0b0, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123d67f4, 0xa000000000371360, 0xa000000000371378, 0x0, 0xe0000000123dcf78) at pipe_read+0x740
dofileread(0xe000000012798d80, 0x6, 0xe0000000123d67d0, 0xa000000000371360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012798d80, 0x6, 0xa000000000371360) at kern_readv+0xa0
sys_read(0xe000000012798d80, 0xa0000000003714e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a0c4a0, 0x140c26000, 0x2000, 0xe000000012798d80, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1263 tid 100080 td 0xe000000012663680
cpu_switch(0xe000000012663680, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012663680, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012663680, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000126639f8, 0xe000000012663680) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000012663680, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa0000000002f9070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000002f9050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000012663680, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000002f92e8, 0xa0000000002f9310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa0000000002f9310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002f90f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000002f9288, 0x200000, 0xe000000012447448, 0x0, 0xa0000000002f9318) at lookup+0xe10
namei(0xa0000000002f9288, 0xe000000012436940, 0x9ffc0000009a5410, 0x0, 0xa0000000002f92f8) at namei+0xb40
vn_open_cred(0xa0000000002f9288, 0xa0000000002f9390, 0x8a8, 0x0, 0xe00000001215c300, 0xe0000000123b4190) at vn_open_cred+0x500
vn_open(0xa0000000002f9288, 0xa0000000002f9390, 0x8a8, 0xe0000000123b4190, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012663680, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe000000012663680, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe000000012663680, 0xa0000000002f94e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe000000012436940, 0x100000, 0x9ffc000000e558b8, 0xe000000012663680, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1262 tid 100094 td 0xe000000012799200
cpu_switch(0xe000000012799200, 0xe000000011151680, 0x9ffc000000eb2ca8, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012799200, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012799200, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000127955a8, 0x5c, 0xe000000012799578, 0xe000000012799200) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000127955a8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000127955a8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000127955a8, 0xe0000000127957e8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123b3794, 0xa000000000369360, 0xa000000000369378, 0x0, 0xe0000000127956b0) at pipe_read+0x740
dofileread(0xe000000012799200, 0x6, 0xe0000000123b3770, 0xa000000000369360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012799200, 0x6, 0xa000000000369360) at kern_readv+0xa0
sys_read(0xe000000012799200, 0xa0000000003694e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe000000012a0c940, 0x140c26000, 0x2000, 0xe000000012799200, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1261 tid 100082 td 0xe000000012662d80
cpu_switch(0xe000000012662d80, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012662d80, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012662d80, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000126630f8, 0xe000000012662d80) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe000000012662d80, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0xe0000000128996b8, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000309070, 0xe000000012899688, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000309050, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012899620, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012899620, 0x200100, 0xe000000012662d80, 0x9ffc000001196448) at vget+0x100
cache_lookup(0xf, 0xa0000000003092e8, 0xa000000000309310, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe000000012447448, 0x0, 0xa000000000309310) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000003090f8, 0x420c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa000000000309288, 0x200000, 0xe000000012447448, 0x0, 0xa000000000309318) at lookup+0xe10
namei(0xa000000000309288, 0xe000000012436000, 0x9ffc0000009a5410, 0x0, 0xa0000000003092f8) at namei+0xb40
vn_open_cred(0xa000000000309288, 0xa000000000309390, 0x8a8, 0x0, 0xe00000001229dd00, 0xe0000000123b28c0) at vn_open_cred+0x500
vn_open(0xa000000000309288, 0xa000000000309390, 0x8a8, 0xe0000000123b28c0, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe000000012662d80, 0xffffffffffffff9c, 0x1400c39a0, 0x0, 0x0, 0xe558b8) at kern_openat+0x490
kern_open(0xe000000012662d80, 0x1400c39a0, 0x0, 0x100000, 0xe558b8) at kern_open+0x40
sys_open(0xe000000012662d80, 0xa0000000003094e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe000000012436000, 0x100000, 0x9ffc000000e558b8, 0xe000000012662d80, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1260 tid 100088 td 0xe000000012433200
cpu_switch(0xe000000012433200, 0xe000000011151680, 0x9ffc000000eb1e70, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012433200, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012433200, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123dda78, 0x5c, 0xe000000012433578, 0xe000000012433200) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123dda78, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123dda78, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123dda78, 0xe0000000123ddcb8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123d69d4, 0xa000000000339360, 0xa000000000339378, 0x0, 0xe0000000123ddb80) at pipe_read+0x740
dofileread(0xe000000012433200, 0x6, 0xe0000000123d69b0, 0xa000000000339360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012433200, 0x6, 0xa000000000339360) at kern_readv+0xa0
sys_read(0xe000000012433200, 0xa0000000003394e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000126e2000, 0x140c26000, 0x2000, 0xe000000012433200, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1259 tid 100068 td 0xe0000000123a6900
cpu_switch(0xe0000000123a6900, 0xe000000011151b00, 0x9ffc000000eb09d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123a6900, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123a6900, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe000000012899688, 0x60, 0xe0000000123a6c78, 0xe0000000123a6900) at sleepq_switch+0x2e0
sleepq_wait(0xe000000012899688, 0x60, 0x9ffc000000d28b50, 0xe0000000123a6900, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe000000012899688, 0x200100, 0
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201403101300.s2AD01QQ080100>