Date: Sat, 17 May 2008 18:34:56 +0200 (CEST) From: freebsd@nagilum.org To: FreeBSD-gnats-submit@FreeBSD.org Cc: freebsd@nagilum.org Subject: i386/123768: [crash] 7.0-STABLE on Soekris net4801 (sys/vm/vm_pageout.c) Message-ID: <20080517163456.A66CE320835@cakebox.homeunix.net> Resent-Message-ID: <200805171650.m4HGo1JI028835@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 123768
>Category: i386
>Synopsis: [crash] 7.0-STABLE on Soekris net4801 (sys/vm/vm_pageout.c)
>Confidential: no
>Severity: serious
>Priority: medium
>Responsible: freebsd-i386
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Sat May 17 16:50:01 UTC 2008
>Closed-Date:
>Last-Modified:
>Originator: Alexander
>Release: FreeBSD 7.0-STABLE i386
>Organization:
private
>Environment:
System: FreeBSD cakebox.tis 7.0-STABLE FreeBSD 7.0-STABLE #5: Thu May 15 22:50:38 CEST 2008 root@cpqak.tis:/usr/obj/usr/src/sys/net4801 i386
Soekris Net4801, FreeBSD STABLE i386 April/May 2008
>Description:
Crash under high network and cpu load.
Here the results of two dumps, I'll collect some more
as long as the problem exists.
I haven't enabled WITNESS since I don't think it will help in this case
but if you think otherwise let me know.
1:
[root@cakebox ~]# kgdb -v /boot/kernel/kernel /var/crash/vmcore.0
kgdb: core file: /var/crash/vmcore.0
kgdb: kernel image: /boot/kernel/kernel
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
There is no member named pathname.
Reading symbols from /boot/kernel/ng_ether.ko...Reading symbols from /boot/kernel/ng_ether.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/ng_ether.ko
Reading symbols from /boot/kernel/netgraph.ko...Reading symbols from /boot/kernel/netgraph.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/netgraph.ko
Reading symbols from /boot/kernel/geom_journal.ko...Reading symbols from /boot/kernel/geom_journal.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/geom_journal.ko
Unread portion of the kernel message buffer:
lockmgr: thread 0xc0fdf440 unlocking unheld lock
panic: vn_finished_write: neg cnt
Uptime: 16h29m13s
Physical memory: 123 MB
Dumping 50 MB: 35 19 3
#0 doadump () at pcpu.h:195
195 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:195
#1 0xc0504669 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:418
#2 0xc0504820 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:572
#3 0xc0574115 in vn_finished_write (mp=0xc0feb538)
at /usr/src/sys/kern/vfs_vnops.c:1049
#4 0xc0637816 in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:1028
#5 0xc04e8d7b in fork_exit (callout=0xc0636a30 <vm_pageout>, arg=0x0,
frame=0xc7b6ad38) at /usr/src/sys/kern/kern_fork.c:783
#6 0xc0646ab0 in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:205
(kgdb) list *0xc0574115
0xc0574115 is in vn_finished_write (/usr/src/sys/kern/vfs_vnops.c:1050).
1045 return;
1046 MNT_ILOCK(mp);
1047 mp->mnt_writeopcount--;
1048 if (mp->mnt_writeopcount < 0)
1049 panic("vn_finished_write: neg cnt");
1050 if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 &&
1051 mp->mnt_writeopcount <= 0)
1052 wakeup(&mp->mnt_writeopcount);
1053 MNT_IUNLOCK(mp);
1054 }
(kgdb) q
2:
root@cakebox ~ > kgdb -v /boot/kernel/kernel /var/crash/vmcore.1
kgdb: core file: /var/crash/vmcore.1
kgdb: kernel image: /boot/kernel/kernel
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
There is no member named pathname.
Reading symbols from /boot/kernel/ng_ether.ko...Reading symbols from /boot/kernel/ng_ether.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/ng_ether.ko
Reading symbols from /boot/kernel/netgraph.ko...Reading symbols from /boot/kernel/netgraph.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/netgraph.ko
Reading symbols from /boot/kernel/geom_journal.ko...Reading symbols from /boot/kernel/geom_journal.ko.symbols...done.
done.
Loaded symbols for /boot/kernel/geom_journal.ko
Unread portion of the kernel message buffer:
panic: lockmgr: thread 0xc0fdf440, not exclusive lock holder 0xc0fdeaa0 unlocking
Uptime: 9h37m37s
Physical memory: 123 MB
Dumping 45 MB: 30 14
#0 doadump () at pcpu.h:195
195 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) bt
#0 doadump () at pcpu.h:195
#1 0xc0504669 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:418
#2 0xc0504820 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:572
#3 0xc04f5a67 in _lockmgr (lkp=0xc1731278, flags=6, interlkp=0xc17312a8,
td=0xc0fdf440, file=0x0, line=0) at /usr/src/sys/kern/kern_lock.c:414
#4 0xc055e519 in vop_stdunlock (ap=0xc7b6ac1c)
at /usr/src/sys/kern/vfs_default.c:280
#5 0xc065fecb in VOP_UNLOCK_APV (vop=0xc06b5dc0, a=0xc7b6ac1c)
at vnode_if.c:1667
#6 0xc056a350 in vput (vp=0xc1731220) at vnode_if.h:877
#7 0xc06377cd in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:1025
#8 0xc04e8d7b in fork_exit (callout=0xc0636a30 <vm_pageout>, arg=0x0,
frame=0xc7b6ad38) at /usr/src/sys/kern/kern_fork.c:783
#9 0xc0646ab0 in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:205
(kgdb) list *0xc06377cd
0xc06377cd is in vm_pageout (/usr/src/sys/vm/vm_pageout.c:1025).
1020 unlock_and_continue:
1021 VM_OBJECT_UNLOCK(object);
1022 if (mp != NULL) {
1023 vm_page_unlock_queues();
1024 if (vp != NULL)
1025 vput(vp);
1026 VFS_UNLOCK_GIANT(vfslocked);
1027 vm_object_deallocate(object);
1028 vn_finished_write(mp);
1029 vm_page_lock_queues();
(kgdb) list *0xc056a350
0xc056a350 is in vput (/usr/src/sys/kern/vfs_subr.c:2195).
2190 error = 0;
2191
2192 if (vp->v_usecount > 1 || ((vp->v_iflag & VI_DOINGINACT) &&
2193 vp->v_usecount == 1)) {
2194 VOP_UNLOCK(vp, 0, td);
2195 v_decr_usecount(vp);
2196 return;
2197 }
2198
2199 if (vp->v_usecount != 1) {
(kgdb) q
root@cakebox ~ > mount
/dev/ad0s1a on / (ufs, NFS exported, local, soft-updates)
devfs on /dev (devfs, local)
procfs on /proc (procfs, local)
devfs on /var/named/dev (devfs, local)
/dev/ad0s1d.eli.journal on /export/home (ufs, asynchronous, NFS exported, local, gjournal)
and dmesg:
Copyright (c) 1992-2008 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD is a registered trademark of The FreeBSD Foundation.
FreeBSD 7.0-STABLE #5: Thu May 15 22:50:38 CEST 2008
root@cpqak.tis:/usr/obj/usr/src/sys/net4801
Timecounter "i8254" frequency 1193182 Hz quality 0
CPU: Geode(TM) Integrated Processor by National Semi (266.63-MHz 586-class CPU)
Origin = "Geode by NSC" Id = 0x540 Stepping = 0
Features=0x808131<FPU,TSC,MSR,CX8,CMOV,MMX>
real memory = 134217728 (128 MB)
avail memory = 126025728 (120 MB)
cryptosoft0: <software crypto> on motherboard
cpu0 on motherboard
pcib0: <Host to PCI bridge> pcibus 0 on motherboard
pci0: <PCI bus> on pcib0
sis0: <NatSemi DP8381[56] 10/100BaseTX> port 0xe100-0xe1ff mem 0xa0000000-0xa000
0fff irq 10 at device 6.0 on pci0
sis0: Silicon Revision: DP83816A
miibus0: <MII bus> on sis0
nsphyter0: <DP83815 10/100 media interface> PHY 0 on miibus0
nsphyter0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
sis0: Ethernet address: 00:00:24:c1:4f:c4
sis0: [ITHREAD]
sis1: <NatSemi DP8381[56] 10/100BaseTX> port 0xe200-0xe2ff mem 0xa0001000-0xa000
1fff irq 10 at device 7.0 on pci0
sis1: Silicon Revision: DP83816A
miibus1: <MII bus> on sis1
nsphyter1: <DP83815 10/100 media interface> PHY 0 on miibus1
nsphyter1: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
sis1: Ethernet address: 00:00:24:c1:4f:c5
sis1: [ITHREAD]
sis2: <NatSemi DP8381[56] 10/100BaseTX> port 0xe300-0xe3ff mem 0xa0002000-0xa0002fff irq 10 at device 8.0 on pci0
sis2: Silicon Revision: DP83816A
miibus2: <MII bus> on sis2
nsphyter2: <DP83815 10/100 media interface> PHY 0 on miibus2
nsphyter2: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
sis2: Ethernet address: 00:00:24:c1:4f:c6
sis2: [ITHREAD]
Geode GPIO@ = 6100
Geode Soekris net4801 comBIOS ver. 1.33 20080103 Copyright (C) 2000-2007
isab0: <PCI-ISA bridge> port 0x6100-0x613f,0x6200-0x623f at device 18.0 on pci0
isa0: <ISA bus> on isab0
pci0: <bridge> at device 18.1 (no driver attached)
atapci0: <National Geode SC1100 ATA33 controller> port 0x1f0-0x1f7,0x3f6,0x170-0x177,0x376,0xe000-0xe00f at device 18.2 on pci0
ata0: <ATA channel 0> on atapci0
ata0: [ITHREAD]
ata1: <ATA channel 1> on atapci0
ata1: [ITHREAD]
Geode CBA@ 0x6000
Geode rev: 06 03
Timecounter "Geode" frequency 27000000 Hz quality 1000
pci0: <bridge> at device 18.5 (no driver attached)
ohci0: <OHCI (generic) USB controller> mem 0xa0003000-0xa0003fff irq 11 at device 19.0 on pci0
ohci0: [GIANT-LOCKED]
ohci0: [ITHREAD]
usb0: OHCI version 1.0, legacy support
usb0: <OHCI (generic) USB controller> on ohci0
usb0: USB revision 1.0
uhub0: <(0x0e11) OHCI root hub, class 9/0, rev 1.00/1.00, addr 1> on usb0
uhub0: 3 ports with 3 removable, self powered
orm0: <ISA Option ROM> at iomem 0xc8000-0xd0fff pnpid ORM0000 on isa0
sio0 at port 0x3f8-0x3ff irq 4 flags 0x10 on isa0
sio0: type 16550A, console
sio0: [FILTER]
sio1 at port 0x2f8-0x2ff irq 3 on isa0
sio1: type 16550A
sio1: [FILTER]
Timecounters tick every 1.000 msec
ad0: 152627MB <SAMSUNG HM160HC LQ100-10> at ata0-master PIO4
Trying to mount root from ufs:/dev/ad0s1a
Accounting enabled
options used to build the kernel+modules:
CPUTYPE?=pentium
CFLAGS=-O -pipe -mtune=pentium -march=pentium
>How-To-Repeat:
Cause high network & CPU load.
For example NFS export / mount it on a fast machine and do a
portupgrade or any other io intensive operation on the nfs mount.
I can also reproduce the problem using a bittorrent client provided
there is enough traffic.
It typically takes between 1-12 hours for the box to crash.
I am also using geli+gjournal for one filesystem on the box but not /.
I am unsure if it relevant.
my kernconf:
cpu I586_CPU
ident NET4801
makeoptions DEBUG=-g # Build kernel with gdb(1) debug symbols
options CPU_GEODE # Geode support
options CPU_SOEKRIS # Soekris support
options SCHED_ULE # ULE scheduler
options PREEMPTION # Enable kernel thread preemption
options INET # InterNETworking
options INET6 # IPv6 communications protocols
options FFS # Berkeley Fast Filesystem
options SOFTUPDATES # Enable FFS soft updates support
options GEOM_ELI # encryption on GEOM providers
options UFS_ACL # Support for access control lists
options UFS_DIRHASH # Improve performance on big directories
options UFS_GJOURNAL # Enable gjournal-based UFS journaling.
options NFSSERVER # Network Filesystem Server
#options MSDOSFS # MSDOS Filesystem
options PROCFS # Process filesystem (requires PSEUDOFS)options PSEUDOFS # Pseudo-filesystem framework
options COMPAT_43TTY # BSD 4.3 TTY compat [KEEP THIS!]
options SCSI_DELAY=5000 # Delay (in ms) before probing SCSI
options KTRACE # ktrace(1) support
options SYSVSHM # SYSV-style shared memory
options SYSVMSG # SYSV-style message queues
options SYSVSEM # SYSV-style semaphores
options _KPOSIX_PRIORITY_SCHEDULING # POSIX P1003_1B real-time extensions
options ADAPTIVE_GIANT # Giant mutex is adaptive.
# Bus support.
device eisa
device pci
# ATA and ATAPI devices
device ata
device atadisk # ATA disk drives
options ATA_STATIC_ID # Static device numbering
# SCSI peripherals
device scbus # SCSI bus (required for SCSI)
device da # Direct Access (disks)
# Serial (COM) ports
device sio # 8250, 16[45]50 based serial ports
# PCI Ethernet NICs that use the common MII bus controller code.
# NOTE: Be sure to keep the 'device miibus' line in order to use these NICs!
device miibus # MII bus support
device sis # Silicon Integrated Systems SiS 900/SiS 7016
# Pseudo devices.
device loop # Network loopback
device random # Entropy device
device ether # Ethernet support
device tun # Packet tunnel.
device pty # Pseudo-ttys (telnet etc)
device md # Memory "disks"
# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
device ether # Ethernet support
device tun # Packet tunnel.
device pty # Pseudo-ttys (telnet etc)
device md # Memory "disks"
# The `bpf' device enables the Berkeley Packet Filter.
# Be aware of the administrative consequences of enabling this!
# Note that 'bpf' is required for DHCP.
device bpf # Berkeley packet filter
device pf # Packet Filter
device pflog # Packet Filter logging
# USB support
device ohci # OHCI PCI->USB interface
device usb # USB Bus (required)
device umass # Disks/Mass storage - Requires scbus and da
device crypto # core crypto support
device cryptodev # /dev/crypto for access to h/w
>Fix:
I wish I knew..
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20080517163456.A66CE320835>