Date: Thu, 6 Mar 2014 10:30:02 GMT From: Anton Shterenlikht <mexas@bris.ac.uk> To: freebsd-ia64@FreeBSD.org Subject: ia64/187219: panic: 11.0-CURRENT #20 r260914: Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks Message-ID: <201403061030.s26AU2Rf026285@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
The following reply was made to PR ia64/187219; it has been noted by GNATS.
From: Anton Shterenlikht <mexas@bris.ac.uk>
To: bug-followup@freebsd.org
Cc:
Subject: ia64/187219: panic: 11.0-CURRENT #20 r260914: Panic String: deadlkres: possible deadlock detected for 0xe000000012079b00, blocked for 1801960 ticks
Date: Thu, 06 Mar 2014 02:28:38 -0800 (PST)
Another deadlock panic with many DEBUG
options removed from the kernel.
# cat info.last
Dump header from device /dev/da2p1
Architecture: ia64
Architecture Version: 1
Dump Length: 67584B (0 MB)
Blocksize: 512
Dumptime: Thu Mar 6 09:58:45 2014
Hostname: mech-as28.men.bris.ac.uk
Magic: FreeBSD Text Dump
Version String: FreeBSD 11.0-CURRENT #22 r260914: Wed Mar 5 10:12:04 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV
Panic String: deadlkres: possible deadlock detected for 0xe000000012651200, blocked for 1801802 ticks
Dump Parity: 3209697066
Bounds: 3
Dump Status: good
#
# cat panic.txt
deadlkres: possible deadlock detected for 0xe000000012651200, blocked for 1801802 ticks
#
# cat version.txt
FreeBSD 11.0-CURRENT #22 r260914: Wed Mar 5 10:12:04 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV
#
# cat config.txt
options CONFIG_AUTOGENERATED
ident ZEEV
machine ia64
cpu ITANIUM2
makeoptions MODULES_OVERRIDE=geom/geom_part opensolaris zfs acl_nfs4 fdescfs
makeoptions DEBUG=-g
options TEXTDUMP_VERBOSE
options TEXTDUMP_PREFERRED
options WITNESS_SKIPSPIN
options WITNESS
options INVARIANT_SUPPORT
options INVARIANTS
options DEADLKRES
options GDB
options DDB
options KDB_TRACE
options KDB
options _KPOSIX_PRIORITY_SCHEDULING
options QUOTA
options UFS_GJOURNAL
options UFS_DIRHASH
options UFS_ACL
options SYSVSHM
options SYSVSEM
options SYSVMSG
options STACK
options SOFTUPDATES
options SMP
options SCTP
options SCSI_DELAY=5000
options SCHED_ULE
options PSEUDOFS
options PROCFS
options PRINTF_BUFR_SIZE=128
options PREEMPTION
options P1003_1B_SEMAPHORES
options NFS_ROOT
options NFSD
options NFSLOCKD
options NFSCL
options MSDOSFS
options MD_ROOT
options MAC
options KTRACE
options INET6
options INET
options INCLUDE_CONFIG_FILE
options GEOM_LABEL
options FFS
options TMPFS
options NULLFS
options FDESCFS
options COMPAT_FREEBSD7
options CD9660
options CAPABILITIES
options CAPABILITY_MODE
options AUDIT
options NEW_PCIB
options GEOM_PART_MBR
options GEOM_PART_GPT
options GEOM_PART_BSD
device acpi
device io
device mem
device uart_ns8250
device miibus
device pci
device scbus
device usb
device isp
device ispfw
device mpt
device cd
device da
device pass
device sa
device ehci
device ohci
device uhci
device umass
device em
device ether
device faith
device firmware
device gif
device loop
device md
device puc
device random
device tun
device uart
device vlan
#
*****************************
msgbuf.txt
*****************************
GDB: debug ports: uart
GDB: current port: uart
KDB: debugger backends: ddb gdb
KDB: current backend: ddb
Copyright (c) 1992-2014 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 11.0-CURRENT #22 r260914: Wed Mar 5 10:12:04 GMT 2014
root@mech-as28.men.bris.ac.uk:/usr/obj/usr/src/sys/ZEEV ia64
gcc version 4.2.1 20070831 patched [FreeBSD]
WARNING: WITNESS option enabled, expect reduced performance.
CPU: Madison II (1600 MHz Itanium 2)
Origin = "GenuineIntel" Revision = 2
Features = 0x1<LB>
real memory = 10737418240 (10240 MB)
avail memory = 10508304384 (10021 MB)
FPSWA Revision = 0x10012, Entry = 0xe0000040ffcc4050
FreeBSD/SMP: Multiprocessor System Detected: 2 CPUs
cpu0: ACPI Id=0, SAPIC Id=0, SAPIC Eid=0 (BSP)
cpu1: ACPI Id=1, SAPIC Id=1, SAPIC Eid=0
random: <Software, Yarrow> initialized
Event timer "ITC" frequency 1600000000 Hz quality 1000
ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe0Block: 32/16 (20130823/tbfadt-603)
ACPI BIOS Warning (bug): 32/64X length mismatch in FADT/Gpe1Block: 32/16 (20130823/tbfadt-603)
acpi0: <HP> on motherboard
acpi0: Power Button (fixed)
acpi0: Sleep Button (fixed)
cpu0: <ACPI CPU> on acpi0
cpu1: <ACPI CPU> on acpi0
Timecounter "ACPI-safe" frequency 3579545 Hz quality 850
acpi_timer0: <32-bit timer at 3.579545MHz> iomem 0xff5c1004-0xff5c1007 on acpi0
acpi_tz0: <Thermal Zone> on acpi0
pcib0: <ACPI Host-PCI bridge> on acpi0
pcib0: could not evaluate _ADR - AE_NOT_FOUND
pci0: <ACPI PCI bus> on pcib0
ohci0: <NEC uPD 9210 USB controller> mem 0x80002000-0x80002fff irq 16 at device 1.0 on pci0
usbus0 on ohci0
ohci1: <NEC uPD 9210 USB controller> mem 0x80001000-0x80001fff irq 17 at device 1.1 on pci0
usbus1 on ohci1
ehci0: <NEC uPD 720100 USB 2.0 controller> mem 0x80000000-0x800000ff irq 18 at device 1.2 on pci0
usbus2: EHCI version 0.95
usbus2 on ehci0
pci0: <mass storage, ATA> at device 2.0 (no driver attached)
pcib1: <ACPI Host-PCI bridge> on acpi0
pcib1: could not evaluate _ADR - AE_NOT_FOUND
pci32: <ACPI PCI bus> on pcib1
mpt0: <LSILogic 1030 Ultra4 Adapter> port 0x2100-0x21ff mem 0x903a0000-0x903bffff,0x90380000-0x9039ffff irq 27 at device 1.0 on pci32
mpt0: MPI Version=1.2.12.0
mpt1: <LSILogic 1030 Ultra4 Adapter> port 0x2000-0x20ff mem 0x90360000-0x9037ffff,0x90340000-0x9035ffff irq 28 at device 1.1 on pci32
mpt1: MPI Version=1.2.12.0
em0: <Intel(R) PRO/1000 Legacy Network Connection 1.0.6> port 0x2240-0x227f mem 0x90320000-0x9033ffff,0x90280000-0x902fffff irq 29 at device 2.0 on pci32
em0: Ethernet address: 00:13:21:5b:05:1c
em1: <Intel(R) PRO/1000 Legacy Network Connection 1.0.6> port 0x2200-0x223f mem 0x90300000-0x9031ffff irq 30 at device 2.1 on pci32
em1: Ethernet address: 00:13:21:5b:05:1d
pcib2: <ACPI Host-PCI bridge> on acpi0
pcib2: could not evaluate _ADR - AE_NOT_FOUND
pci64: <ACPI PCI bus> on pcib2
pcib3: <ACPI Host-PCI bridge> on acpi0
pcib3: could not evaluate _ADR - AE_NOT_FOUND
pci96: <ACPI PCI bus> on pcib3
pcib4: <ACPI Host-PCI bridge> on acpi0
pcib4: could not evaluate _ADR - AE_NOT_FOUND
pci128: <ACPI PCI bus> on pcib4
pcib5: <ACPI Host-PCI bridge> on acpi0
pcib5: could not evaluate _ADR - AE_NOT_FOUND
pci192: <ACPI PCI bus> on pcib5
isp0: <Qlogic ISP 2422 PCI FC-AL Adapter> port 0xc000-0xc0ff mem 0xe0040000-0xe0040fff irq 71 at device 1.0 on pci192
pcib6: <ACPI Host-PCI bridge> on acpi0
pcib6: could not evaluate _ADR - AE_NOT_FOUND
pci224: <ACPI PCI bus> on pcib6
uart0: <HP Auxiliary Diva Serial Port> mem 0xf8051000-0xf805100f irq 82 at device 1.0 on pci224
puc0: <HP Diva Serial [GSP] Multiport UART - Everest SP2> mem 0xf8050000-0xf8050fff,0xf8020000-0xf803ffff irq 82 at device 1.1 on pci224
uart1: <Non-standard ns8250 class UART with FIFOs> at port 1 on puc0
uart1: console (9600,n,8,1)
uart2: <Non-standard ns8250 class UART with FIFOs> at port 3 on puc0
vgapci0: <VGA-compatible display> port 0xe000-0xe0ff mem 0xf0000000-0xf7ffffff,0xf8040000-0xf804ffff at device 2.0 on pci224
vgapci0: Boot video device
uart3: <16550 or compatible> iomem 0xff5e0000-0xff5e0007 irq 34 on acpi0
uart4: <16550 or compatible> iomem 0xff5e2000-0xff5e2007 irq 35 on acpi0
uart4: debug port (9600,n,8,1)
Timecounters tick every 1.000 msec
usbus0: 12Mbps Full Speed USB v1.0
usbus1: 12Mbps Full Speed USB v1.0
usbus2: 480Mbps High Speed USB v2.0
ugen1.1: <NEC> at usbus1
ugen0.1: <NEC> at usbus0
uhub0: <NEC OHCI root HUB, class 9/0, rev 1.00/1.00, addr 1> on usbus1
uhub1: <NEC OHCI root HUB, class 9/0, rev 1.00/1.00, addr 1> on usbus0
ugen2.1: <NEC> at usbus2
uhub2: <NEC EHCI root HUB, class 9/0, rev 2.00/1.00, addr 1> on usbus2
random: unblocking device.
uhub0: 2 ports with 2 removable, self powered
uhub1: 3 ports with 3 removable, self powered
uhub2: 5 ports with 5 removable, self powered
pass1 at isp0 bus 0 scbus2 target 0 lun 0
da1 at isp0 bus 0 scbus2 target 0 lun 1
pass1: <COMPAQ MSA1000 4.32> Fixed Storage Array SCSI-4 device
da1: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
pass1: Serial Number P56350GX3RX04Z
da1: Serial Number P56350GX3RX04Z
pass1: 200.000MB/s transfersda1: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 WWPN 0x500805f3000ec221 PortID 0x10000
PortID 0x10000
pass1: Command Queueing enabled
da1: Command Queueing enabled
da1: 69460MB (142255575 512 byte sectors: 255H 63S/T 8855C)
da2 at isp0 bus 0 scbus2 target 0 lun 2
da2: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da2: Serial Number P56350GX3RX04Z
da2: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da2: Command Queueing enabled
da2: 69460MB (142255575 512 byte sectors: 255H 63S/T 8855C)
da3 at isp0 bus 0 scbus2 target 0 lun 5
da3: <COMPAQ MSA1000 VOLUME 4.32> Fixed Direct Access SCSI-4 device
da3: Serial Number P56350GX3RX04Z
da3: 200.000MB/s transfers WWNN 0x500805f3000ec220 WWPN 0x500805f3000ec221 PortID 0x10000
da3: Command Queueing enabled
da3: 140011MB (286744185 512 byte sectors: 255H 63S/T 17849C)
da0 at mpt0 bus 0 scbus0 target 0 lun 0
da0: <HP 73.4G ST373454LC HPC2> Fixed Direct Access SCSI-3 device
da0: Serial Number 3KP05D0W00007532H5SA
da0: 320.000MB/s transfers (160.000MHz, offset 63, 16bit)
da0: Command Queueing enabled
da0: 70007MB (143374738 512 byte sectors: 255H 63S/T 8924C)
WARNING: WITNESS option enabled, expect reduced performance.
Trying to mount root from ufs:/dev/da0p2 [rw]...
WARNING: / was not properly dismounted
<118>Setting hostuuid: ee652761-3dcc-11df-9074-0013215b051c.
<118>Setting hostid: 0x8968724b.
<118>Starting ddb.
<118>Entropy harvesting: interrupts ethernet point_to_point swi.
<118>Starting file system checks:
<118>/dev/da0p2: 562514 files, 3749278 used, 30867146 free (305122 frags, 3820253 blocks, 0.9% fragmentation)
<118>/dev/da1p1: 2152 files, 81832 used, 17141529 free (465 frags, 2142633 blocks, 0.0% fragmentation)
<118>/dev/da3p1: 6077 files, 1328197 used, 68099306 free (6234 frags, 8511634 blocks, 0.0% fragmentation)
<118>Mounting local file systems:.
<118>Writing entropy file:.
<118>Setting hostname: mech-as28.men.bris.ac.uk.
<118>Starting Network: lo0 em0 em1.
<118>lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
<118> options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
<118> inet6 ::1 prefixlen 128
<118> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x3
<118> inet 127.0.0.1 netmask 0xff000000
<118> nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
<118>em0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC>
<118> ether 00:13:21:5b:05:1c
<118> inet 137.222.187.28 netmask 0xffffff00 broadcast 137.222.187.255
<118> inet6 fe80::213:21ff:fe5b:51c%em0 prefixlen 64 scopeid 0x1
<118> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
<118> media: Ethernet autoselect
<118> status: no carrier
<118>em1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 1500
<118> options=209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC>
<118> ether 00:13:21:5b:05:1d
<118> inet 10.10.10.14 netmask 0xffffff00 broadcast 10.10.10.255
<118> inet6 fe80::213:21ff:fe5b:51d%em1 prefixlen 64 scopeid 0x2
<118> nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
<118> media: Ethernet autoselect (100baseTX <full-duplex>)
<118> status: active
<118>Starting devd.
<118>Starting pflogd:
<118>add net default: gateway 137.222.187.250
<118>add net fe80::: gateway ::1
<118>add net ff02::: gateway ::1
<118>add net ::ffff:0.0.0.0: gateway ::1
<118>add net ::0.0.0.0: gateway ::1
<118>ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib
<118>Creating and/or trimming log files.
<118>Starting syslogd.
<118>savecore: reboot after panic: deadlkres: possible deadlock detected for 0xe000000011781680, blocked for 1800556 ticks
<118>Mar 6 09:26:13 mech-as28 savecore: reboot after panic: deadlkres: possible deadlock detected for 0xe000000011781680, blocked for 1800556 ticks
<118>savecore: writing core to /var/crash/textdump.tar.2
<118>/var/crash/vmcore.2 not found
<118>Starting casperd.
lock order reversal:
1st 0xa00000009e049220 bufwait (bufwait) @ /usr/src/sys/kern/vfs_bio.c:3081
2nd 0xe000000012136c00 dirhash (dirhash) @ /usr/src/sys/ufs/ufs/ufs_dirhash.c:284
KDB: stack backtrace:
db_trace_self(0x9ffc0000000d89c0) at db_trace_self+0x40
db_trace_self_wrapper(0x9ffc000000572870) at db_trace_self_wrapper+0x70
kdb_backtrace(0x9ffc000000e2b2d8, 0x9ffc0000005a8990) at kdb_backtrace+0xc0
_witness_debugger(0x1, 0x9ffc000000d2c138, 0x9ffc0000005ac370, 0xb9d, 0x9ffc000000d59270) at _witness_debugger+0x60
witness_checkorder(0xe000000012136c00, 0x9ffc000000d2ba80, 0x9ffc000000d59270, 0x11c, 0x0) at witness_checkorder+0x15c0
_sx_xlock(0xe000000012136c00, 0x0, 0x9ffc000000d59270, 0x11c) at _sx_xlock+0x120
ufsdirhash_acquire(0xe0000000123f5500, 0xe000000012136c00, 0x9ffc000000975d60) at ufsdirhash_acquire+0x50
ufsdirhash_add(0xe0000000123f5500, 0xa0000000002f1098, 0x934, 0xa0000000002f1040, 0x9ffc00000097dbf0, 0xda1) at ufsdirhash_add+0x20
ufs_direnter(0xe000000012664760, 0xe0000000126641d8, 0xa0000000002f1098, 0xa0000000002f1318, 0xa00000009e049720, 0x0) at ufs_direnter+0x1260
ufs_mkdir(0xa0000000002f1370, 0xa0000000002f1098, 0xe000000012664760) at ufs_mkdir+0x14f0
VOP_MKDIR_APV(0x9ffc000000df2660, 0xa0000000002f1370, 0xa0000000002f1318, 0x0, 0x9ffc0000006650c0, 0x818, 0x9ffc0000010015a0) at VOP_MKDIR_APV+0x220
kern_mkdirat(0xe000000012384900, 0xffffffffffffff9c, 0x140804040, 0x0, 0x1c0) at kern_mkdirat+0x3a0
kern_mkdir(0xe000000012384900, 0x140804040, 0x0, 0x1c0) at kern_mkdir+0x40
sys_mkdir(0xe000000012384900, 0xa0000000002f14e8, 0x9ffc000000a2cc80, 0x48d) at sys_mkdir+0x30
syscall(0xe00000001244ade0, 0x1c0, 0x9ffc000000a29630, 0xe000000012384900, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
<118>Clearing /tmp (X related).
<118>Recovering vi editor sessions:.
<118>Updating motd:.
<118>Mounting late file systems:.
<118>Starting ntpd.
<118>Starting bsdstats.
<118>Posting monthly OS statistics to rpt.bsdstats.org
<118>Performing sanity check on sshd configuration.
<118>Starting sshd.
<118>Starting sendmail_submit.
<118>Starting sendmail_msp_queue.
<118>Starting cron.
<118>Local package initialization:cd: /var/portbuild/: No such file or directory
<118>.
<118>Starting inetd.
<118>Starting background file system checks in 60 seconds.
<118>
<118>Thu Mar 6 09:26:22 GMT 2014
<118>Mar 6 09:26:58 mech-as28 su: mexas to root on /dev/pts/0
panic: deadlkres: possible deadlock detected for 0xe000000012651200, blocked for 1801802 ticks
cpuid = 0
KDB: stack backtrace:
db_trace_self(0x9ffc0000000d89c0) at db_trace_self+0x40
db_trace_self_wrapper(0x9ffc000000572870) at db_trace_self_wrapper+0x70
kdb_backtrace(0x9ffc000000e2b2d8, 0x9ffc0000004f99e0, 0x40c, 0x9ffc0000010015a0) at kdb_backtrace+0xc0
vpanic(0x9ffc000000d16c10, 0xa00000009dfff518) at vpanic+0x260
panic(0x9ffc000000d16c10, 0x9ffc000000d16e68, 0xe000000012651200, 0x1b7e4a) at panic+0x80
deadlkres(0x9ffc000000d16ba0, 0xe000000012651200, 0x9ffc000000dc8358, 0x9ffc000000d16ba0) at deadlkres+0x700
fork_exit(0x9ffc000000d6b130, 0x0, 0xa00000009dfff550) at fork_exit+0x120
enter_userland() at enter_userland
KDB: enter: panic
*************************************
ddb.txt
*************************************
db:0:kdb.enter.panic> run lockinfo
db:1:lockinfo> show locks
db:1:locks> show alllocks
Process 1062 (atrun) thread 0xe000000012769680 (100100)
shared lockmgr ufs (ufs) r = 0 (0xe000000012425c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1061 (sh) thread 0xe000000012766d80 (100099)
shared lockmgr ufs (ufs) r = 0 (0xe0000000120f1c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1058 (atrun) thread 0xe0000000128cc480 (100096)
shared lockmgr ufs (ufs) r = 0 (0xe000000012425c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1056 (atrun) thread 0xe0000000123fe480 (100073)
shared lockmgr ufs (ufs) r = 0 (0xe000000012425c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1054 (sh) thread 0xe000000012650900 (100083)
shared lockmgr ufs (ufs) r = 0 (0xe0000000120f1c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1052 (atrun) thread 0xe00000001237e000 (100070)
shared lockmgr ufs (ufs) r = 0 (0xe000000012425c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1050 (atrun) thread 0xe0000000123fed80 (100071)
shared lockmgr ufs (ufs) r = 0 (0xe000000012425c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1048 (dd) thread 0xe00000001237f680 (100065)
exclusive lockmgr bufwait (bufwait) r = 0 (0xa00000009e03d9c0) locked @ /usr/src/sys/kern/vfs_bio.c:3081
exclusive lockmgr ufs (ufs) r = 0 (0xe000000012838068) locked @ /usr/src/sys/ufs/ffs/ffs_vfsops.c:1711
exclusive lockmgr ufs (ufs) r = 0 (0xe0000000124245f0) locked @ /usr/src/sys/kern/vfs_lookup.c:518
Process 1045 (atrun) thread 0xe000000012768000 (100091)
exclusive lockmgr bufwait (bufwait) r = 0 (0xa00000009e06a348) locked @ /usr/src/sys/kern/vfs_bio.c:2345
exclusive lockmgr ufs (ufs) r = 0 (0xe0000000128914b0) locked @ /usr/src/sys/ufs/ffs/ffs_vfsops.c:1711
shared lockmgr ufs (ufs) r = 0 (0xe000000012425c10) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1043 (fsck_ufs) thread 0xe000000012385b00 (100075)
exclusive lockmgr ufs (ufs) r = 0 (0xe0000000124425f0) locked @ /usr/src/sys/kern/vfs_subr.c:2101
Process 1042 (grep) thread 0xe000000012384480 (100062)
exclusive lockmgr bufwait (bufwait) r = 0 (0xa00000009e06a8d0) locked @ /usr/src/sys/kern/vfs_bio.c:2345
shared lockmgr ufs (ufs) r = 0 (0xe0000000128392d8) locked @ /usr/src/sys/kern/vfs_vnops.c:704
Process 1004 (login) thread 0xe000000011691b00 (100051)
exclusive lockmgr ufs (ufs) r = 0 (0xe000000012424d50) locked @ /usr/src/sys/kern/vfs_vnops.c:373
Process 20 (syncer) thread 0xe00000001231c900 (100059)
exclusive lockmgr ufs (ufs) r = 0 (0xe000000012645de8) locked @ /usr/src/sys/kern/vfs_subr.c:2101
exclusive lockmgr syncer (syncer) r = 0 (0xe000000012425a38) locked @ /usr/src/sys/kern/vfs_subr.c:1720
db:1:alllocks> show lockedvnods
Locked vnodes
db:0:kdb.enter.panic> show pcpu
cpuid = 0
dynamic pcpu = 0x40040040ff1df980
curthread = 0xe000000011780480: pid 0 "deadlkres"
curpcb = 0
fpcurthread = none
idlethread = 0xe000000011151b00: tid 100003 "idle: cpu0"
MD: vhpt = 0xe0000040ff000000
MD: lid = 0
MD: clock = 0x31b2d05669a
MD: clock_mode = 2
MD: clock_load = 0
MD: stats = 0x9ffc0000011a4a30
MD: pmap = 0x9ffc000000e53920
spin locks held:
db:0:kdb.enter.panic> bt
Tracing pid 0 tid 100049 td 0xe000000011780480
kdb_enter(0x9ffc000000d21190, 0x9ffc000000d21190, 0x9ffc0000004f9a30, 0x40c) at kdb_enter+0x92
vpanic(0x9ffc000000d16c10, 0xa00000009dfff518) at vpanic+0x2b0
panic(0x9ffc000000d16c10, 0x9ffc000000d16e68, 0xe000000012651200, 0x1b7e4a) at panic+0x80
deadlkres(0x9ffc000000d16ba0, 0xe000000012651200, 0x9ffc000000dc8358, 0x9ffc000000d16ba0) at deadlkres+0x700
fork_exit(0x9ffc000000d6b130, 0x0, 0xa00000009dfff550) at fork_exit+0x120
enter_userland() at enter_userland
db:0:kdb.enter.panic> ps
pid ppid pgrp uid state wmesg wchan cmd
1062 1060 1062 0 Ds ufs 0xe0000000128914b0 atrun
1061 1059 1061 2 Ds ufs 0xe0000000124245f0 sh
1060 944 944 0 S piperd 0xe0000000123aba78 cron
1059 944 944 0 S piperd 0xe00000001289f0d8 cron
1058 1057 1058 0 Ds ufs 0xe0000000128914b0 atrun
1057 944 944 0 S piperd 0xe0000000123ab5a8 cron
1056 1055 1056 0 Ds ufs 0xe0000000128914b0 atrun
1055 944 944 0 S piperd 0xe00000001289f5a8 cron
1054 1053 1054 2 Ds ufs 0xe0000000124245f0 sh
1053 944 944 0 S piperd 0xe0000000126a55a8 cron
1052 1051 1052 0 Ds ufs 0xe0000000128914b0 atrun
1051 944 944 0 S piperd 0xe0000000123aae70 cron
1050 1049 1050 0 Ds ufs 0xe0000000128914b0 atrun
1049 944 944 0 S piperd 0xe0000000126a50d8 cron
1048 1047 1047 2 D biowr 0xa00000009e03d938 dd
1047 1046 1047 2 Ss wait 0xe00000001244a000 sh
1046 944 944 0 S piperd 0xe0000000126a4c08 cron
1045 1044 1045 0 Ds biord 0xa00000009e06a2c0 atrun
1044 944 944 0 S piperd 0xe00000001289fa78 cron
1043 999 23 0 D getblk 0xa00000009e032160 fsck_ufs
1042 1039 1042 0 D+ biord 0xa00000009e06a848 grep
1039 1038 1039 0 S+ wait 0xe00000001244cde0 sh
1038 1034 1038 1001 S+ wait 0xe0000000126ae4a0 su
1034 1033 1034 1001 Ss+ pause 0xe00000001244d328 tcsh
1033 1031 1031 1001 S select 0xe000000012241640 sshd
1031 933 1031 0 Ss select 0xe000000012241a40 sshd
1004 1 1004 0 Ds+ getblk 0xa00000009e032160 login
1003 1 1 0 S ttydcd 0xe00000001173b0e8 getty
1000 1 23 0 S+ piperd 0xe0000000123aa9a0 logger
999 1 23 0 S+ wait 0xe0000000123f84a0 fsck
983 1 983 0 Ss select 0xe000000012242040 inetd
944 1 944 0 Ss nanslp 0x9ffc000000e60598 cron
939 1 939 25 Ss pause 0xe00000001244ae88 sendmail
936 1 936 0 Ss select 0xe000000012241440 sendmail
933 1 933 0 Ss select 0xe00000001178a0c0 sshd
866 1 866 0 Ss select 0xe00000001178b8c0 ntpd
802 1 802 0 Ss select 0xe00000001178b1c0 casperd
801 1 23 0 S+ select 0xe000000011699e40 casperd
772 1 772 0 Ds ufs 0xe0000000124425f0 syslogd
647 1 647 0 Ss select 0xe000000011736bc0 devd
138 1 138 0 Ss pause 0xe00000001244b328 adjkerntz
22 0 0 0 DL sdflush 0x9ffc000000e2c480 [softdepflush]
21 0 0 0 DL vlruwt 0xe000000012318de0 [vnlru]
20 0 0 0 DL getblk 0xa00000009e032160 [syncer]
19 0 0 0 DL psleep 0x9ffc000000e2b860 [bufdaemon]
18 0 0 0 DL pgzero 0x9ffc000000e2c7ec [pagezero]
17 0 0 0 DL psleep 0x9ffc000000e2c754 [vmdaemon]
9 0 0 0 DL psleep 0x9ffc000000e2c784 [pagedaemon]
6 0 0 0 DL waiting_ 0x9ffc000001198d70 [sctp_iterator]
5 0 0 0 DL ispf 0xe00000001173c800 [isp0: fc_thrd0]
4 0 0 0 DL idle 0xa000000000a22000 [mpt_recovery1]
3 0 0 0 DL idle 0xa000000000a0a000 [mpt_recovery0]
16 0 0 0 DL (threaded) [usb]
100039 D - 0xa000000000a06e18 [usbus2]
100038 D - 0xa000000000a06dc0 [usbus2]
100037 D - 0xa000000000a06d68 [usbus2]
100036 D - 0xa000000000a06d10 [usbus2]
100034 D - 0xa0000000009fd460 [usbus1]
100033 D - 0xa0000000009fd408 [usbus1]
100032 D - 0xa0000000009fd3b0 [usbus1]
100031 D - 0xa0000000009fd358 [usbus1]
100029 D - 0xa0000000009f9460 [usbus0]
100028 D - 0xa0000000009f9408 [usbus0]
100027 D - 0xa0000000009f93b0 [usbus0]
100026 D - 0xa0000000009f9358 [usbus0]
15 0 0 0 DL tzpoll 0x9ffc000000e2a288 [acpi_thermal]
2 0 0 0 DL (threaded) [cam]
100054 D ccb_scan 0x9ffc000000e2cb38 [scanner]
100015 D - 0x9ffc000000e2cc80 [doneq0]
14 0 0 0 DL - 0x9ffc000000e2a578 [rand_harvestq]
13 0 0 0 DL (threaded) [geom]
100011 D - 0x9ffc000000e2aa28 [g_down]
100010 D - 0x9ffc000000e2aa20 [g_up]
100009 D - 0x9ffc000000e2aa10 [g_event]
12 0 0 0 WL (threaded) [intr]
100048 I [swi0: uart uart+++]
100046 I [irq71: isp0]
100042 I [irq28: mpt1]
100040 I [irq27: mpt0]
100035 I [irq18: ehci0]
100030 I [irq17: ohci1]
100025 I [irq16: ohci0]
100023 I [swi6: Giant taskq]
100021 I [swi5: fast taskq]
100014 I [swi6: task queue]
100008 I [swi1: netisr 0]
100007 I [swi3: vm]
100006 I [swi4: clock]
100005 I [swi4: clock]
11 0 0 0 RL (threaded) [idle]
100004 Run CPU 1 [idle: cpu1]
100003 CanRun [idle: cpu0]
1 0 1 0 SLs wait 0xe00000001114e4a0 [init]
10 0 0 0 DL audit_wo 0x9ffc00000119d408 [audit]
0 0 0 0 RLs (threaded) [kernel]
100053 D - 0xe000000011142300 [CAM taskq]
100049 Run CPU 0 [deadlkres]
100045 D - 0xe000000011207e00 [em1 taskq]
100044 D - 0xe000000011208200 [em0 taskq]
100022 D - 0xe000000011141a00 [thread taskq]
100020 D - 0xe000000011141d00 [kqueue taskq]
100019 D - 0xe000000011141e00 [ffs_trim taskq]
100018 D - 0xe000000011142000 [acpi_task_2]
100017 D - 0xe000000011142000 [acpi_task_1]
100016 D - 0xe000000011142000 [acpi_task_0]
100012 D - 0xe000000011143100 [firmware taskq]
100000 D swapin 0x9ffc000000e52ee8 [swapper]
db:0:kdb.enter.panic> alltrace
Tracing command atrun pid 1062 tid 100100 td 0xe000000012769680
cpu_switch(0xe000000012769680, 0xe000000012769b00, 0x9ffc000000eb1ec0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012769680, 0xe000000012769b00, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012769680, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000128914b0, 0x60, 0xe0000000127699f8, 0xe000000012769680) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000128914b0, 0x60, 0x9ffc000000d28b50, 0xe000000012769680, 0x9ffc0000004c43a0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000128914b0, 0x200100, 0xe0000000128914e0, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0x16b0
ffs_lock(0xa000000000399140, 0xe0000000128914b0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000399120, 0x8) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012891448, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012891448, 0x200100, 0xe000000012769680, 0x57) at vget+0x100
vfs_hash_get(0xe000000012346330, 0x1c2a80, 0xe000000012891448, 0xe000000012769680, 0xa000000000399198, 0x0, 0x0, 0x9ffc000000d353d8) at vfs_hash_get+0x180
ffs_vgetf(0xe000000012346330, 0x1c2a80, 0x200000, 0xa000000000399198, 0x0) at ffs_vgetf+0x50
ffs_vget(0xe000000012346330, 0x1c2a80, 0x200000, 0xa000000000399198, 0x9ffc00000097c2b0, 0x1836) at ffs_vget+0x40
ufs_lookup_ino(0xe000000012425ba8, 0xa000000000399310, 0xa000000000399338, 0x0) at ufs_lookup_ino+0x1cd0
ufs_lookup(0xa0000000003991e8, 0x9ffc000000a42240, 0x38a) at ufs_lookup+0x40
VOP_CACHEDLOOKUP_APV(0x9ffc000000df2660, 0xa0000000003991d0, 0xa000000000399338, 0x0, 0x9ffc000000630460) at VOP_CACHEDLOOKUP_APV+0x220
vfs_cache_lookup(0xe000000012425ba8, 0x0, 0xa000000000399338) at vfs_cache_lookup+0x1f0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa000000000399208, 0x4004144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000003992b0, 0x200000, 0xe000000012425ba8, 0x0, 0xa000000000399340) at lookup+0xe10
namei(0xa0000000003992b0, 0xe0000000128c3280, 0x4000144, 0x0, 0xa000000000399320, 0xa000000000399368, 0xa000000000399360, 0xa000000000399310) at namei+0xb40
kern_chdir(0xe000000012769680, 0x100004d10, 0x0) at kern_chdir+0x80
sys_chdir(0xe000000012769680, 0xa0000000003994e8, 0x9ffc000000a2cc80, 0x48d) at sys_chdir+0x30
syscall(0xe0000000128c3280, 0x7fffffffffffed40, 0x100004e38, 0xe000000012769680, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1061 tid 100099 td 0xe000000012766d80
cpu_switch(0xe000000012766d80, 0xe000000011151b00, 0x9ffc000000eb2168, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012766d80, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012766d80, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000124245f0, 0x60, 0xe0000000127670f8, 0xe000000012766d80) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000124245f0, 0x60, 0x9ffc000000d28b50, 0xe000000012766d80, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000124245f0, 0x200100, 0xe000000012424620, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000391090, 0xe0000000124245f0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000391070, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012424588, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012424588, 0x200100, 0xe000000012766d80, 0x9ffc000001196448) at vget+0x100
cache_lookup(0x7, 0xa000000000391220, 0xa000000000391248, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe0000000120f1ba8, 0x0, 0xa000000000391248) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa000000000391118, 0x400c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000003911c0, 0x200000, 0xe0000000120f1ba8, 0x0, 0xa000000000391250) at lookup+0xe10
namei(0xa0000000003911c0, 0xe0000000128e1280, 0x4000144, 0x0, 0xa000000000391230, 0xa000000000391278, 0xa000000000391270, 0xa000000000391220) at namei+0xb40
kern_statat_vnhook(0xe000000012766d80, 0x0, 0xffffffffffffff9c, 0x140c0ab10, 0x0, 0xa000000000391320, 0x0) at kern_statat_vnhook+0xf0
kern_statat(0xe000000012766d80, 0x0, 0xffffffffffffff9c, 0x140c0ab10, 0x0, 0xa000000000391320) at kern_statat+0x40
kern_stat(0xe000000012766d80, 0x140c0ab10, 0x0, 0xa000000000391320) at kern_stat+0x40
sys_stat(0xe000000012766d80, 0xa0000000003914e8, 0x9ffc000000a2cc80, 0x48d) at sys_stat+0x30
syscall(0xe0000000128e1280, 0x7fffffffffffe708, 0x14031d640, 0xe000000012766d80, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1060 tid 100098 td 0xe000000012769b00
cpu_switch(0xe000000012769b00, 0xe000000011151680, 0x9ffc000000eb2370, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012769b00, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012769b00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123aba78, 0x5c, 0xe000000012769e78, 0xe000000012769b00) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123aba78, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123aba78, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123aba78, 0xe0000000123abcb8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000126f7014, 0xa000000000389360, 0xa000000000389378, 0x0, 0xe0000000123abb80) at pipe_read+0x740
dofileread(0xe000000012769b00, 0x6, 0xe0000000126f6ff0, 0xa000000000389360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012769b00, 0x6, 0xa000000000389360) at kern_readv+0xa0
sys_read(0xe000000012769b00, 0xa0000000003894e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000128c3720, 0x140c26000, 0x2000, 0xe000000012769b00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1059 tid 100097 td 0xe0000000128cc000
cpu_switch(0xe0000000128cc000, 0xe000000011151680, 0x9ffc000000eb0b60, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000128cc000, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000128cc000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe00000001289f0d8, 0x5c, 0xe0000000128cc378, 0xe0000000128cc000) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001289f0d8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001289f0d8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe00000001289f0d8, 0xe00000001289f318, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe00000001238de24, 0xa000000000381360, 0xa000000000381378, 0x0, 0xe00000001289f1e0) at pipe_read+0x740
dofileread(0xe0000000128cc000, 0x6, 0xe00000001238de00, 0xa000000000381360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000128cc000, 0x6, 0xa000000000381360) at kern_readv+0xa0
sys_read(0xe0000000128cc000, 0xa0000000003814e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000128c8000, 0x140c26000, 0x2000, 0xe0000000128cc000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command atrun pid 1058 tid 100096 td 0xe0000000128cc480
cpu_switch(0xe0000000128cc480, 0xe000000011151b00, 0x9ffc000000eb1ec0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000128cc480, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000128cc480, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000128914b0, 0x60, 0xe0000000128cc7f8, 0xe0000000128cc480) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000128914b0, 0x60, 0x9ffc000000d28b50, 0xe0000000128cc480, 0x9ffc0000004c43a0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000128914b0, 0x200100, 0xe0000000128914e0, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0x16b0
ffs_lock(0xa000000000379140, 0xe0000000128914b0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000379120, 0x8) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012891448, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012891448, 0x200100, 0xe0000000128cc480, 0x57) at vget+0x100
vfs_hash_get(0xe000000012346330, 0x1c2a80, 0xe000000012891448, 0xe0000000128cc480, 0xa000000000379198, 0x0, 0x0, 0x9ffc000000d353d8) at vfs_hash_get+0x180
ffs_vgetf(0xe000000012346330, 0x1c2a80, 0x200000, 0xa000000000379198, 0x0) at ffs_vgetf+0x50
ffs_vget(0xe000000012346330, 0x1c2a80, 0x200000, 0xa000000000379198, 0x9ffc00000097c2b0, 0x1836) at ffs_vget+0x40
ufs_lookup_ino(0xe000000012425ba8, 0xa000000000379310, 0xa000000000379338, 0x0) at ufs_lookup_ino+0x1cd0
ufs_lookup(0xa0000000003791e8, 0x9ffc000000a42240, 0x38a) at ufs_lookup+0x40
VOP_CACHEDLOOKUP_APV(0x9ffc000000df2660, 0xa0000000003791d0, 0xa000000000379338, 0x0, 0x9ffc000000630460) at VOP_CACHEDLOOKUP_APV+0x220
vfs_cache_lookup(0xe000000012425ba8, 0x0, 0xa000000000379338) at vfs_cache_lookup+0x1f0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa000000000379208, 0x4004144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000003792b0, 0x200000, 0xe000000012425ba8, 0x0, 0xa000000000379340) at lookup+0xe10
namei(0xa0000000003792b0, 0xe0000000128c84a0, 0x4000144, 0x0, 0xa000000000379320, 0xa000000000379368, 0xa000000000379360, 0xa000000000379310) at namei+0xb40
kern_chdir(0xe0000000128cc480, 0x100004d10, 0x0) at kern_chdir+0x80
sys_chdir(0xe0000000128cc480, 0xa0000000003794e8, 0x9ffc000000a2cc80, 0x48d) at sys_chdir+0x30
syscall(0xe0000000128c84a0, 0x7fffffffffffed40, 0x100004e38, 0xe0000000128cc480, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1057 tid 100095 td 0xe000000012767200
cpu_switch(0xe000000012767200, 0xe000000011151680, 0x9ffc000000eb09a8, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012767200, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012767200, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123ab5a8, 0x5c, 0xe000000012767578, 0xe000000012767200) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123ab5a8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123ab5a8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123ab5a8, 0xe0000000123ab7e8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000126f7104, 0xa000000000371360, 0xa000000000371378, 0x0, 0xe0000000123ab6b0) at pipe_read+0x740
dofileread(0xe000000012767200, 0x6, 0xe0000000126f70e0, 0xa000000000371360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012767200, 0x6, 0xa000000000371360) at kern_readv+0xa0
sys_read(0xe000000012767200, 0xa0000000003714e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000128e1720, 0x140c26000, 0x2000, 0xe000000012767200, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command atrun pid 1056 tid 100073 td 0xe0000000123fe480
cpu_switch(0xe0000000123fe480, 0xe000000011151680, 0x9ffc000000eb1ec0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123fe480, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123fe480, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000128914b0, 0x60, 0xe0000000123fe7f8, 0xe0000000123fe480) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000128914b0, 0x60, 0x9ffc000000d28b50, 0xe0000000123fe480, 0x9ffc0000004c43a0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000128914b0, 0x200100, 0xe0000000128914e0, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0x16b0
ffs_lock(0xa0000000002c1140, 0xe0000000128914b0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000002c1120, 0x8) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012891448, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012891448, 0x200100, 0xe0000000123fe480, 0x57) at vget+0x100
vfs_hash_get(0xe000000012346330, 0x1c2a80, 0xe000000012891448, 0xe0000000123fe480, 0xa0000000002c1198, 0x0, 0x0, 0x9ffc000000d353d8) at vfs_hash_get+0x180
ffs_vgetf(0xe000000012346330, 0x1c2a80, 0x200000, 0xa0000000002c1198, 0x0) at ffs_vgetf+0x50
ffs_vget(0xe000000012346330, 0x1c2a80, 0x200000, 0xa0000000002c1198, 0x9ffc00000097c2b0, 0x1836) at ffs_vget+0x40
ufs_lookup_ino(0xe000000012425ba8, 0xa0000000002c1310, 0xa0000000002c1338, 0x0) at ufs_lookup_ino+0x1cd0
ufs_lookup(0xa0000000002c11e8, 0x9ffc000000a42240, 0x38a) at ufs_lookup+0x40
VOP_CACHEDLOOKUP_APV(0x9ffc000000df2660, 0xa0000000002c11d0, 0xa0000000002c1338, 0x0, 0x9ffc000000630460) at VOP_CACHEDLOOKUP_APV+0x220
vfs_cache_lookup(0xe000000012425ba8, 0x0, 0xa0000000002c1338) at vfs_cache_lookup+0x1f0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002c1208, 0x4004144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000002c12b0, 0x200000, 0xe000000012425ba8, 0x0, 0xa0000000002c1340) at lookup+0xe10
namei(0xa0000000002c12b0, 0xe00000001231ade0, 0x4000144, 0x0, 0xa0000000002c1320, 0xa0000000002c1368, 0xa0000000002c1360, 0xa0000000002c1310) at namei+0xb40
kern_chdir(0xe0000000123fe480, 0x100004d10, 0x0) at kern_chdir+0x80
sys_chdir(0xe0000000123fe480, 0xa0000000002c14e8, 0x9ffc000000a2cc80, 0x48d) at sys_chdir+0x30
syscall(0xe00000001231ade0, 0x7fffffffffffed40, 0x100004e38, 0xe0000000123fe480, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1055 tid 100094 td 0xe0000000128cc900
cpu_switch(0xe0000000128cc900, 0xe000000011151b00, 0x9ffc000000eb13a8, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000128cc900, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000128cc900, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe00000001289f5a8, 0x5c, 0xe0000000128ccc78, 0xe0000000128cc900) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001289f5a8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001289f5a8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe00000001289f5a8, 0xe00000001289f7e8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe00000001238df14, 0xa000000000369360, 0xa000000000369378, 0x0, 0xe00000001289f6b0) at pipe_read+0x740
dofileread(0xe0000000128cc900, 0x6, 0xe00000001238def0, 0xa000000000369360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000128cc900, 0x6, 0xa000000000369360) at kern_readv+0xa0
sys_read(0xe0000000128cc900, 0xa0000000003694e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000128c8940, 0x140c26000, 0x2000, 0xe0000000128cc900, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1054 tid 100083 td 0xe000000012650900
cpu_switch(0xe000000012650900, 0xe000000011151b00, 0x9ffc000000eb2168, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012650900, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012650900, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000124245f0, 0x60, 0xe000000012650c78, 0xe000000012650900) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000124245f0, 0x60, 0x9ffc000000d28b50, 0xe000000012650900, 0x9ffc0000004c37d0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000124245f0, 0x200100, 0xe000000012424620, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0xae0
ffs_lock(0xa000000000311090, 0xe0000000124245f0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa000000000311070, 0x9ffc0000004f6150) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012424588, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012424588, 0x200100, 0xe000000012650900, 0x9ffc000001196448) at vget+0x100
cache_lookup(0x7, 0xa000000000311220, 0xa000000000311248, 0x0, 0x0) at cache_lookup+0xf70
vfs_cache_lookup(0xe0000000120f1ba8, 0x0, 0xa000000000311248) at vfs_cache_lookup+0x1a0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa000000000311118, 0x400c144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000003111c0, 0x200000, 0xe0000000120f1ba8, 0x0, 0xa000000000311250) at lookup+0xe10
namei(0xa0000000003111c0, 0xe0000000126af720, 0x4000144, 0x0, 0xa000000000311230, 0xa000000000311278, 0xa000000000311270, 0xa000000000311220) at namei+0xb40
kern_statat_vnhook(0xe000000012650900, 0x0, 0xffffffffffffff9c, 0x140c0ab10, 0x0, 0xa000000000311320, 0x0) at kern_statat_vnhook+0xf0
kern_statat(0xe000000012650900, 0x0, 0xffffffffffffff9c, 0x140c0ab10, 0x0, 0xa000000000311320) at kern_statat+0x40
kern_stat(0xe000000012650900, 0x140c0ab10, 0x0, 0xa000000000311320) at kern_stat+0x40
sys_stat(0xe000000012650900, 0xa0000000003114e8, 0x9ffc000000a2cc80, 0x48d) at sys_stat+0x30
syscall(0xe0000000126af720, 0x7fffffffffffe708, 0x14031d640, 0xe000000012650900, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1053 tid 100068 td 0xe00000001237e900
cpu_switch(0xe00000001237e900, 0xe000000011151680, 0x9ffc000000eb2ca8, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe00000001237e900, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001237e900, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000126a55a8, 0x5c, 0xe00000001237ec78, 0xe00000001237e900) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000126a55a8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000126a55a8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000126a55a8, 0xe0000000126a57e8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000126f71f4, 0xa000000000299360, 0xa000000000299378, 0x0, 0xe0000000126a56b0) at pipe_read+0x740
dofileread(0xe00000001237e900, 0x6, 0xe0000000126f71d0, 0xa000000000299360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe00000001237e900, 0x6, 0xa000000000299360) at kern_readv+0xa0
sys_read(0xe00000001237e900, 0xa0000000002994e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000123f8940, 0x140c26000, 0x2000, 0xe00000001237e900, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command atrun pid 1052 tid 100070 td 0xe00000001237e000
cpu_switch(0xe00000001237e000, 0xe000000011151b00, 0x9ffc000000eb1ec0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe00000001237e000, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001237e000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000128914b0, 0x60, 0xe00000001237e378, 0xe00000001237e000) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000128914b0, 0x60, 0x9ffc000000d28b50, 0xe00000001237e000, 0x9ffc0000004c43a0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000128914b0, 0x200100, 0xe0000000128914e0, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0x16b0
ffs_lock(0xa0000000002a9140, 0xe0000000128914b0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000002a9120, 0x8) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012891448, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012891448, 0x200100, 0xe00000001237e000, 0x57) at vget+0x100
vfs_hash_get(0xe000000012346330, 0x1c2a80, 0xe000000012891448, 0xe00000001237e000, 0xa0000000002a9198, 0x0, 0x0, 0x9ffc000000d353d8) at vfs_hash_get+0x180
ffs_vgetf(0xe000000012346330, 0x1c2a80, 0x200000, 0xa0000000002a9198, 0x0) at ffs_vgetf+0x50
ffs_vget(0xe000000012346330, 0x1c2a80, 0x200000, 0xa0000000002a9198, 0x9ffc00000097c2b0, 0x1836) at ffs_vget+0x40
ufs_lookup_ino(0xe000000012425ba8, 0xa0000000002a9310, 0xa0000000002a9338, 0x0) at ufs_lookup_ino+0x1cd0
ufs_lookup(0xa0000000002a91e8, 0x9ffc000000a42240, 0x38a) at ufs_lookup+0x40
VOP_CACHEDLOOKUP_APV(0x9ffc000000df2660, 0xa0000000002a91d0, 0xa0000000002a9338, 0x0, 0x9ffc000000630460) at VOP_CACHEDLOOKUP_APV+0x220
vfs_cache_lookup(0xe000000012425ba8, 0x0, 0xa0000000002a9338) at vfs_cache_lookup+0x1f0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002a9208, 0x4004144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000002a92b0, 0x200000, 0xe000000012425ba8, 0x0, 0xa0000000002a9340) at lookup+0xe10
namei(0xa0000000002a92b0, 0xe0000000123f8000, 0x4000144, 0x0, 0xa0000000002a9320, 0xa0000000002a9368, 0xa0000000002a9360, 0xa0000000002a9310) at namei+0xb40
kern_chdir(0xe00000001237e000, 0x100004d10, 0x0) at kern_chdir+0x80
sys_chdir(0xe00000001237e000, 0xa0000000002a94e8, 0x9ffc000000a2cc80, 0x48d) at sys_chdir+0x30
syscall(0xe0000000123f8000, 0x7fffffffffffed40, 0x100004e38, 0xe00000001237e000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1051 tid 100064 td 0xe00000001237fb00
cpu_switch(0xe00000001237fb00, 0xe000000011151680, 0x9ffc000000eb27d0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe00000001237fb00, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001237fb00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000123aae70, 0x5c, 0xe00000001237fe78, 0xe00000001237fb00) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000123aae70, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000123aae70, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000123aae70, 0xe0000000123ab0b0, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000126f72e4, 0xa000000000279360, 0xa000000000279378, 0x0, 0xe0000000123aaf78) at pipe_read+0x740
dofileread(0xe00000001237fb00, 0x6, 0xe0000000126f72c0, 0xa000000000279360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe00000001237fb00, 0x6, 0xa000000000279360) at kern_readv+0xa0
sys_read(0xe00000001237fb00, 0xa0000000002794e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe00000001114ede0, 0x140c26000, 0x2000, 0xe00000001237fb00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command atrun pid 1050 tid 100071 td 0xe0000000123fed80
cpu_switch(0xe0000000123fed80, 0xe000000011151b00, 0x9ffc000000eb1ec0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123fed80, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123fed80, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000128914b0, 0x60, 0xe0000000123ff0f8, 0xe0000000123fed80) at sleepq_switch+0x2e0
sleepq_wait(0xe0000000128914b0, 0x60, 0x9ffc000000d28b50, 0xe0000000123fed80, 0x9ffc0000004c43a0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xe0000000128914b0, 0x200100, 0xe0000000128914e0, 0x9ffc000000d153b8, 0x60, 0x0, 0x9ffc000000d363a0, 0x835) at __lockmgr_args+0x16b0
ffs_lock(0xa0000000002b1140, 0xe0000000128914b0, 0x200100) at ffs_lock+0x130
VOP_LOCK1_APV(0x9ffc000000df1d20, 0xa0000000002b1120, 0x8) at VOP_LOCK1_APV+0x230
_vn_lock(0xe000000012891448, 0x200100, 0x9ffc000000d363a0, 0x835) at _vn_lock+0x100
vget(0xe000000012891448, 0x200100, 0xe0000000123fed80, 0x57) at vget+0x100
vfs_hash_get(0xe000000012346330, 0x1c2a80, 0xe000000012891448, 0xe0000000123fed80, 0xa0000000002b1198, 0x0, 0x0, 0x9ffc000000d353d8) at vfs_hash_get+0x180
ffs_vgetf(0xe000000012346330, 0x1c2a80, 0x200000, 0xa0000000002b1198, 0x0) at ffs_vgetf+0x50
ffs_vget(0xe000000012346330, 0x1c2a80, 0x200000, 0xa0000000002b1198, 0x9ffc00000097c2b0, 0x1836) at ffs_vget+0x40
ufs_lookup_ino(0xe000000012425ba8, 0xa0000000002b1310, 0xa0000000002b1338, 0x0) at ufs_lookup_ino+0x1cd0
ufs_lookup(0xa0000000002b11e8, 0x9ffc000000a42240, 0x38a) at ufs_lookup+0x40
VOP_CACHEDLOOKUP_APV(0x9ffc000000df2660, 0xa0000000002b11d0, 0xa0000000002b1338, 0x0, 0x9ffc000000630460) at VOP_CACHEDLOOKUP_APV+0x220
vfs_cache_lookup(0xe000000012425ba8, 0x0, 0xa0000000002b1338) at vfs_cache_lookup+0x1f0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa0000000002b1208, 0x4004144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000002b12b0, 0x200000, 0xe000000012425ba8, 0x0, 0xa0000000002b1340) at lookup+0xe10
namei(0xa0000000002b12b0, 0xe00000001231b720, 0x4000144, 0x0, 0xa0000000002b1320, 0xa0000000002b1368, 0xa0000000002b1360, 0xa0000000002b1310) at namei+0xb40
kern_chdir(0xe0000000123fed80, 0x100004d10, 0x0) at kern_chdir+0x80
sys_chdir(0xe0000000123fed80, 0xa0000000002b14e8, 0x9ffc000000a2cc80, 0x48d) at sys_chdir+0x30
syscall(0xe00000001231b720, 0x7fffffffffffed40, 0x100004e38, 0xe0000000123fed80, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1049 tid 100080 td 0xe000000012651680
cpu_switch(0xe000000012651680, 0xe000000011151680, 0x9ffc000000eb1a60, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012651680, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012651680, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000126a50d8, 0x5c, 0xe0000000126519f8, 0xe000000012651680) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000126a50d8, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000126a50d8, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000126a50d8, 0xe0000000126a5318, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000126f73d4, 0xa0000000002f9360, 0xa0000000002f9378, 0x0, 0xe0000000126a51e0) at pipe_read+0x740
dofileread(0xe000000012651680, 0x6, 0xe0000000126f73b0, 0xa0000000002f9360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe000000012651680, 0x6, 0xa0000000002f9360) at kern_readv+0xa0
sys_read(0xe000000012651680, 0xa0000000002f94e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe00000001244a940, 0x140c26000, 0x2000, 0xe000000012651680, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command dd pid 1048 tid 100065 td 0xe00000001237f680
cpu_switch(0xe00000001237f680, 0xe000000011151680, 0x9ffc000000eb2848, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe00000001237f680, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe00000001237f680, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xa00000009e03d938, 0x5c, 0xe00000001237f9f8, 0xe00000001237f680) at sleepq_switch+0x2e0
sleepq_wait(0xa00000009e03d938, 0x5c, 0x9ffc000000d28b50, 0xe00000001237f680, 0x9ffc00000050be60) at sleepq_wait+0xb0
_sleep(0xa00000009e03d938, 0xe000000010c20ad0, 0x5c, 0x9ffc000000d331a8, 0x0, 0x0, 0x100) at _sleep+0x7d0
bwait(0xa00000009e03d938, 0x5c, 0x9ffc000000d331a8) at bwait+0xc0
bufwait(0xa00000009e03d938, 0x9ffc000000628270, 0x38a) at bufwait+0x80
bufwrite(0xa00000009e03d938, 0xa000, 0x0, 0x0, 0x9ffc000000960d10, 0x30d, 0x9ffc0000010015a0) at bufwrite+0x3a0
ffs_bufwrite(0xa00000009e03d938, 0x9ffc000000d58388, 0x0, 0x9ffc0000009165a0, 0x12ad, 0x0) at ffs_bufwrite+0x520
ffs_update(0xe000000012838000, 0x1, 0xa00000009e03d938, 0xe0000000123ec588, 0xe00000001213f0b8, 0xe0000000127b4ab0) at ffs_update+0x8b0
ufs_makeinode(0x8100, 0xe000000012424588, 0xa0000000002812e8, 0xa000000000281310) at ufs_makeinode+0x820
ufs_create(0xa000000000281258, 0x9ffc000000a45e80, 0x38a) at ufs_create+0x80
VOP_CREATE_APV(0x9ffc000000df2660, 0xa000000000281240, 0xa000000000281310, 0x0, 0x9ffc0000006762f0) at VOP_CREATE_APV+0x220
vn_open_cred(0xa000000000281288, 0xa000000000281390, 0x100, 0x0, 0xe000000012355600, 0xe0000000126f7450) at vn_open_cred+0x330
vn_open(0xa000000000281288, 0xa000000000281390, 0x100, 0xe0000000126f7450, 0x9ffc00000066bfb0, 0x695, 0x0) at vn_open+0x50
kern_openat(0xe00000001237f680, 0xffffffffffffff9c, 0x140806043, 0x0, 0x0, 0x1b6) at kern_openat+0x490
kern_open(0xe00000001237f680, 0x140806043, 0x0, 0x602, 0x1b6) at kern_open+0x40
sys_open(0xe00000001237f680, 0xa0000000002814e8, 0x9ffc000000a2cc80, 0x48d) at sys_open+0x40
syscall(0xe0000000123f9720, 0x602, 0x1b6, 0xe00000001237f680, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command sh pid 1047 tid 100082 td 0xe000000012650d80
cpu_switch(0xe000000012650d80, 0xe000000011151b00, 0x9ffc000000eb1e20, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012650d80, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012650d80, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe00000001244a000, 0x6c, 0xe0000000126510f8, 0xe000000012650d80) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001244a000, 0x6c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001244a000, 0x6c, 0x9ffc000000d57ae8, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe00000001244a000, 0xe00000001244a100, 0x6c, 0x9ffc000000d57ae8, 0x0, 0x0, 0x100, 0x0) at _sleep+0x7b0
kern_wait6(0xe000000012650d80, 0x7, 0x0, 0xa000000000309390, 0x30, 0x0, 0x0) at kern_wait6+0xba0
kern_wait(0xe000000012650d80, 0xffffffff, 0xa000000000309390, 0x0, 0x0) at kern_wait+0xe0
sys_wait4(0xe000000012650d80, 0xa0000000003094e8, 0xa000000000309390, 0x9ffc000000a2cc80) at sys_wait4+0x70
syscall(0xe00000001244a000, 0x7fffffffffffe8e0, 0x0, 0xe000000012650d80, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1046 tid 100087 td 0xe0000000123ffb00
cpu_switch(0xe0000000123ffb00, 0xe000000011151680, 0x9ffc000000eb0fc0, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123ffb00, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123ffb00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe0000000126a4c08, 0x5c, 0xe0000000123ffe78, 0xe0000000123ffb00) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe0000000126a4c08, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe0000000126a4c08, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe0000000126a4c08, 0xe0000000126a4e48, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe00000001238c2f4, 0xa000000000331360, 0xa000000000331378, 0x0, 0xe0000000126a4d10) at pipe_read+0x740
dofileread(0xe0000000123ffb00, 0x6, 0xe00000001238c2d0, 0xa000000000331360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000123ffb00, 0x6, 0xa000000000331360) at kern_readv+0xa0
sys_read(0xe0000000123ffb00, 0xa0000000003314e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe0000000126ae940, 0x140c26000, 0x2000, 0xe0000000123ffb00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command atrun pid 1045 tid 100091 td 0xe000000012768000
cpu_switch(0xe000000012768000, 0xe000000011151b00, 0x9ffc000000eb1470, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012768000, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012768000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xa00000009e06a2c0, 0x5c, 0xe000000012768378, 0xe000000012768000) at sleepq_switch+0x2e0
sleepq_wait(0xa00000009e06a2c0, 0x5c, 0x9ffc000000d28b50, 0xe000000012768000, 0x9ffc00000050be60) at sleepq_wait+0xb0
_sleep(0xa00000009e06a2c0, 0xe000000010c203b0, 0x5c, 0x9ffc000000d331a0, 0x0, 0x0, 0x100) at _sleep+0x7d0
bwait(0xa00000009e06a2c0, 0x5c, 0x9ffc000000d331a0) at bwait+0xc0
bufwait(0xa00000009e06a2c0, 0x9ffc00000062b7a0, 0x591, 0x0) at bufwait+0x50
breadn_flags(0xe0000000123ec588, 0xa00000009e06a2c0, 0xa00000009e06a3b8, 0x0, 0x0, 0x0, 0x0, 0x0) at breadn_flags+0x300
ffs_vgetf(0xe000000012346330, 0x1c2a80, 0x80000, 0xa000000000351198, 0x0) at ffs_vgetf+0x6d0
ffs_vget(0xe000000012346330, 0x1c2a80, 0x200000, 0xa000000000351198, 0x9ffc00000097c2b0, 0x1836) at ffs_vget+0x40
ufs_lookup_ino(0xe000000012425ba8, 0xa000000000351310, 0xa000000000351338, 0x0) at ufs_lookup_ino+0x1cd0
ufs_lookup(0xa0000000003511e8, 0x9ffc000000a42240, 0x38a) at ufs_lookup+0x40
VOP_CACHEDLOOKUP_APV(0x9ffc000000df2660, 0xa0000000003511d0, 0xa000000000351338, 0x0, 0x9ffc000000630460) at VOP_CACHEDLOOKUP_APV+0x220
vfs_cache_lookup(0xe000000012425ba8, 0x0, 0xa000000000351338) at vfs_cache_lookup+0x1f0
VOP_LOOKUP_APV(0x9ffc000000df2660, 0xa000000000351208, 0x4004144, 0x0) at VOP_LOOKUP_APV+0x220
lookup(0xa0000000003512b0, 0x200000, 0xe000000012425ba8, 0x0, 0xa000000000351340) at lookup+0xe10
namei(0xa0000000003512b0, 0xe00000001244d720, 0x4000144, 0x0, 0xa000000000351320, 0xa000000000351368, 0xa000000000351360, 0xa000000000351310) at namei+0xb40
kern_chdir(0xe000000012768000, 0x100004d10, 0x0) at kern_chdir+0x80
sys_chdir(0xe000000012768000, 0xa0000000003514e8, 0x9ffc000000a2cc80, 0x48d) at sys_chdir+0x30
syscall(0xe00000001244d720, 0x7fffffffffffed40, 0x100004e38, 0xe000000012768000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command cron pid 1044 tid 100074 td 0xe0000000123fe000
cpu_switch(0xe0000000123fe000, 0xe000000011151b00, 0x9ffc000000eb1970, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe0000000123fe000, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe0000000123fe000, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xe00000001289fa78, 0x5c, 0xe0000000123fe378, 0xe0000000123fe000) at sleepq_switch+0x2e0
sleepq_catch_signals(0xe00000001289fa78, 0x5c, 0x9ffc000000d28b50, 0x9ffc000000d28f10) at sleepq_catch_signals+0x190
sleepq_wait_sig(0xe00000001289fa78, 0x5c, 0x9ffc000000d2c5a0, 0x100, 0x9ffc00000050be40) at sleepq_wait_sig+0x30
_sleep(0xe00000001289fa78, 0xe00000001289fcb8, 0x5c, 0x9ffc000000d2c5a0, 0x0, 0x0, 0x100) at _sleep+0x7b0
pipe_read(0xe0000000123a4114, 0xa0000000002c9360, 0xa0000000002c9378, 0x0, 0xe00000001289fb80) at pipe_read+0x740
dofileread(0xe0000000123fe000, 0x6, 0xe0000000123a40f0, 0xa0000000002c9360, 0x0, 0x0) at dofileread+0x150
kern_readv(0xe0000000123fe000, 0x6, 0xa0000000002c9360) at kern_readv+0xa0
sys_read(0xe0000000123fe000, 0xa0000000002c94e8, 0x9ffc000000a2cc80, 0x48d) at sys_read+0x100
syscall(0xe00000001244c940, 0x140c26000, 0x2000, 0xe0000000123fe000, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command fsck_ufs pid 1043 tid 100075 td 0xe000000012385b00
cpu_switch(0xe000000012385b00, 0xe000000011151680, 0x9ffc000000eb0f48, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012385b00, 0xe000000011151680, 0x9ffc000000e7b518, 0x9ffc000000e7b500) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012385b00, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xa00000009e032160, 0x60, 0xe000000012385e78, 0xe000000012385b00) at sleepq_switch+0x2e0
sleepq_wait(0xa00000009e032160, 0x60, 0x9ffc000000d28b50, 0xe000000012385b00, 0x9ffc0000004c43a0, 0x12ab) at sleepq_wait+0xb0
__lockmgr_args(0xa00000009e032160, 0x81900, 0xe0000000123ec658, 0x9ffc000000d16b98, 0x60, 0x0, 0x9ffc000000d32db0, 0xc09) at __lockmgr_args+0x16b0
getblk(0xe0000000123ec588, 0x7b81d00, 0x8000, 0x0, 0x0, 0x0) at getblk+0x2b0
breadn_flags(0xe0000000123ec588, 0x7b81d00, 0x8000, 0x0, 0x0, 0x0, 0x0, 0x0) at breadn_flags+0x40
ffs_update(0xe000000012442588, 0x0, 0xf70378, 0xe0000000123ec588, 0xe00000001213f0b8, 0xe00000001264eff0, 0xe00000001264efc0, 0xe00000001213f000) at ffs_update+0x3e0
ffs_syncvnode(0xe000000012442588, 0x0, 0x0, 0x0, 0x9ffc000000d58b68, 0xe000000012442670) at ffs_syncvnode+0x810
ffs_sync(0xe000000012346330, 0x2, 0xe000000012442588, 0x9ffc000000664710) at ffs_sync+0x670
sys_sync(0x0, 0x9ffc000000d37be0, 0x9ffc000000e278d8, 0xe000000012385be4) at sys_sync+0x350
syscall(0xe00000001244c4a0, 0x7fffffffffffed58, 0x0, 0xe000000012385b00, 0x0, 0x0, 0x9ffc000000a29620, 0x8) at syscall+0x5e0
epc_syscall_return() at epc_syscall_return
Tracing command grep pid 1042 tid 100062 td 0xe000000012384480
cpu_switch(0xe000000012384480, 0xe000000011151b00, 0x9ffc000000eb2820, 0x9ffc0000005532f0) at cpu_switch+0xd0
sched_switch(0xe000000012384480, 0xe000000011151b00, 0x9ffc000000e7a818, 0x9ffc000000e7a800) at sched_switch+0x890
mi_switch(0x104, 0x0, 0xe000000012384480, 0x9ffc00000058c3c0) at mi_switch+0x3f0
sleepq_switch(0xa00000009e06a848, 0x5c, 0xe0000000123847f8, 0xe000000012384480) at sleepq_switch+0x2e0
sleepq_wait(0xa00000009e06a848, 0x5c, 0x9ffc000000d28b50, 0xe000000012384480, 0x9ffc00000050be60) at sleepq_wait+0xb0
_sleep(0xa00000009e06a848, 0xe000000010c205f0, 0x5c, 0x9ffc000000d331a0, 0x0, 0x0, 0x100) at _sleep+0x7d0
bwait(0xa00000009e06a848, 0x5c, 0x9ffc000000d331a0) at bwait+0xc0
bufwait(0xa00000009e06a848, 0x9ffc00000062b7a0, 0x591, 0x0) at bufwait+0x50
breadn_flags(0xe000000012839270, 0xa00000009e06a848, 0xa00000009e06a940, 0x0, 0x0, 0x0, 0x0, 0x8) at breadn_flags+0x300
ffs_read(0xa000000000269240, 0x1000, 0x8000) at ffs_read+0x540
VOP_READ_APV(0x9ffc000000df1d20, 0xa000000000269240, 0xe000000012839270, 0x0) at VOP_READ_APV+0x220
vn_read(0xe0000000123a4dc0, 0xa000000000269360
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201403061030.s26AU2Rf026285>