Date: Wed, 19 Jul 2006 09:14:28 GMT From: hunreal <hunreal@gmail.com> To: freebsd-gnats-submit@FreeBSD.org Subject: misc/100525: Panic while running webalizer Message-ID: <200607190914.k6J9ESUX043855@www.freebsd.org> Resent-Message-ID: <200607190920.k6J9KCtZ032438@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 100525
>Category: misc
>Synopsis: Panic while running webalizer
>Confidential: no
>Severity: non-critical
>Priority: medium
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Wed Jul 19 09:20:07 GMT 2006
>Closed-Date:
>Last-Modified:
>Originator: hunreal
>Release: 6.1-RELEASE
>Organization:
hshh.org
>Environment:
FreeBSD box.hshh.org 6.1-RELEASE-p3 FreeBSD 6.1-RELEASE-p3 #0: Wed Jul 19 15:05:42 CST 2006 root@box.hshh.org:/usr/obj/usr/src/sys/SMP i386
>Description:
While I am running webalizer to process a large web log file (>600M), it panic.
It was twice, but not every time, I don't know how to repeat it. webalizer is compiled by ports. Kernel is compiled by origin SMP configure file, no any /etc/make.conf items nor CFLAGS.
Attached kgdb and nm result.
[root@www] /data/dump# kgdb /usr/obj/usr/src/sys/SMP/kernel.debug vmcore.0
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
Unread portion of the kernel message buffer:
kernel trap 12 with interrupts disabled
Fatal trap 12: page fault while in kernel mode
cpuid = 1; apic id = 01
fault virtual address = 0x18c5c06c
fault code = supervisor read, page not present
instruction pointer = 0x20:0xc0647bbd
stack pointer = 0x28:0xe35fcbd8
frame pointer = 0x28:0xe35fcbe4
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, def32 1, gran 1
processor eflags = resume, IOPL = 0
current process = 34 (pagedaemon)
trap number = 12
panic: page fault
cpuid = 1
Uptime: 39m12s
Dumping 1023 MB (2 chunks)
chunk 0: 1MB (159 pages) ... ok
chunk 1: 1023MB (261872 pages) 1007 991 975 959 943 927 911 895 879 863 847 831 815 799 783 767 751 735 719 703 687 671 655 639 623 607 591 575 559 543 527 511 495 479 463 447 431 415 399 383 367 351 335 319 303 287 271 255 239 223 207 191 175 159 143 127 111 95 79 63 47 31 15
#0 doadump () at pcpu.h:165
165 pcpu.h: No such file or directory.
in pcpu.h
(kgdb) where
#0 doadump () at pcpu.h:165
#1 0xc06507c9 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:402
#2 0xc0650af1 in panic (fmt=0xc08958ae "%s") at /usr/src/sys/kern/kern_shutdown.c:558
#3 0xc084a2cc in trap_fatal (frame=0xe35fcb98, eva=415613036) at /usr/src/sys/i386/i386/trap.c:836
#4 0xc0849a3e in trap (frame=
{tf_fs = -484769784, tf_es = -1056571352, tf_ds = 40, tf_edi = -995140992, tf_esi = 415612780, tf_ebp = -480261148, tf_isp = -480261180, tf_ebx = -977200595, tf_edx = 415612781, tf_ecx = 415612783, tf_eax = 1, tf_trapno = 12, tf_err = 0, tf_eip = -1067156547, tf_cs = 32, tf_eflags = 65542, tf_esp = -977200595, tf_ss = -1055522752})
at /usr/src/sys/i386/i386/trap.c:269
#5 0xc0836c4a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
#6 0xc0647bbd in _mtx_lock_sleep (m=0xc5c11a2d, tid=3299826304, opts=0, file=0x0, line=0)
at /usr/src/sys/kern/kern_mutex.c:532
#7 0xc07cbb78 in vm_pageout_fallback_object_lock (m=0xc1160040, next=0x1) at /usr/src/sys/vm/vm_pageout.c:250
#8 0xc07cc87f in vm_pageout_scan (pass=0) at /usr/src/sys/vm/vm_pageout.c:803
#9 0xc07cdb8a in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:1554
#10 0xc063a32d in fork_exit (callout=0xc07cd878 <vm_pageout>, arg=0x0, frame=0xe35fcd38)
at /usr/src/sys/kern/kern_fork.c:805
#11 0xc0836cac in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:208
(kgdb) bt full
#0 doadump () at pcpu.h:165
No locals.
#1 0xc06507c9 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:402
first_buf_printf = 1
#2 0xc0650af1 in panic (fmt=0xc08958ae "%s") at /usr/src/sys/kern/kern_shutdown.c:558
td = (struct thread *) 0xc4af5a80
bootopt = 260
newpanic = 0
ap = 0xc4af5a80 "$F拍`#"
buf = "page fault", '\0' <repeats 245 times>
#3 0xc084a2cc in trap_fatal (frame=0xe35fcb98, eva=415613036) at /usr/src/sys/i386/i386/trap.c:836
code = 40
type = 12
ss = 40
esp = 0
softseg = {ssd_base = 0, ssd_limit = 1048575, ssd_type = 27, ssd_dpl = 0, ssd_p = 1, ssd_xx = 0, ssd_xx1 = 0,
ssd_def32 = 1, ssd_gran = 1}
msg = 0x0
#4 0xc0849a3e in trap (frame=
{tf_fs = -484769784, tf_es = -1056571352, tf_ds = 40, tf_edi = -995140992, tf_esi = 415612780, tf_ebp = -480261148, tf_isp = -480261180, tf_ebx = -977200595, tf_edx = 415612781, tf_ecx = 415612783, tf_eax = 1, tf_trapno = 12, tf_err = 0, tf_eip = -1067156547, tf_cs = 32, tf_eflags = 65542, tf_esp = -977200595, tf_ss = -1055522752})
at /usr/src/sys/i386/i386/trap.c:269
td = (struct thread *) 0xc4af5a80
p = (struct proc *) 0xc4c54624
sticks = 1
i = 0
ucode = 0
type = 12
code = 0
eva = 415613036
#5 0xc0836c4a in calltrap () at /usr/src/sys/i386/i386/exception.s:139
No locals.
#6 0xc0647bbd in _mtx_lock_sleep (m=0xc5c11a2d, tid=3299826304, opts=0, file=0x0, line=0)
at /usr/src/sys/kern/kern_mutex.c:532
owner = (struct thread *) 0x18c5bf6c
v = 415612781
#7 0xc07cbb78 in vm_pageout_fallback_object_lock (m=0xc1160040, next=0x1) at /usr/src/sys/vm/vm_pageout.c:250
marker = {pageq = {tqe_next = 0xc149f688, tqe_prev = 0xc1160040}, listq = {tqe_next = 0x0, tqe_prev = 0x0},
---Type <return> to continue, or q <return> to quit---
left = 0x0, right = 0x0, object = 0x0, pindex = 0, phys_addr = 0, md = {pv_list_count = 0, pv_list = {tqh_first = 0x0,
tqh_last = 0x0}}, queue = 33, flags = 4105, pc = 0, wire_count = 1, cow = 0, hold_count = 0, act_count = 0 '\0',
busy = 0 '\0', valid = 0 '\0', dirty = 0 '\0'}
unchanged = 415612783
queue = 33
object = 0xc5c11a2d
#8 0xc07cc87f in vm_pageout_scan (pass=0) at /usr/src/sys/vm/vm_pageout.c:803
m = 0xc1160040
next = 0xc149f688
marker = {pageq = {tqe_next = 0x0, tqe_prev = 0x0}, listq = {tqe_next = 0x0, tqe_prev = 0x0}, left = 0x0,
right = 0x0, object = 0x0, pindex = 0, phys_addr = 0, md = {pv_list_count = 0, pv_list = {tqh_first = 0x0,
tqh_last = 0x0}}, queue = 33, flags = 4105, pc = 0, wire_count = 1, cow = 0, hold_count = 0, act_count = 0 '\0',
busy = 0 '\0', valid = 0 '\0', dirty = 0 '\0'}
page_shortage = 3584
maxscan = 83854
pcount = 83854
addl_page_shortage = 0
addl_page_shortage_init = 0
p = (struct proc *) 0xc1160040
bigproc = (struct proc *) 0xc4af5a80
td = (struct thread *) 0x18c5bf6d
size = 83854
bigsize = 3299826572
object = 0xc5c11a2d
actcount = 0
cache_cur = 83854
cache_first_failure = 2351
cache_last_free = 25
vnodes_skipped = 0
maxlaunder = 32
#9 0xc07cdb8a in vm_pageout () at /usr/src/sys/vm/vm_pageout.c:1554
error = 1
pass = 0
#10 0xc063a32d in fork_exit (callout=0xc07cd878 <vm_pageout>, arg=0x0, frame=0xe35fcd38)
at /usr/src/sys/kern/kern_fork.c:805
p = (struct proc *) 0xc4c54624
td = (struct thread *) 0x18c5bf6d
#11 0xc0836cac in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:208
No locals.
[root@www] /data/logs# nm -n /boot/kernel/kernel | grep c0647b
c0647b20 T _mtx_lock_sleep
>How-To-Repeat:
>Fix:
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200607190914.k6J9ESUX043855>