Date: Thu, 14 Oct 2004 02:14:36 -0400 From: Mike Tancsa <mike@sentex.net> To: freebsd-stable@freebsd.org Subject: Repeated panics... in the serial code ? Message-ID: <6.1.2.0.0.20041014021404.04143d00@64.7.153.2>
next in thread | raw e-mail | index | archive | help
We have a number of RELENG_4 boxes that are dumping core with crash dumps
generally in the same place. Unfortunately, we are not sure just yet how
to repeat the dump and sometimes we do not get a full dump as the hardware
watchdog interrupts the dump process :(.
The boxes are running a 4 port PCI serial card (lava) but we have seen a
similar issue with a Titan serial card as well.
IdlePTD at physical address 0x003f5000
initial pcb at physical address 0x003493c0
panicstr: free: multiple frees
panic messages:
---
panic: free: multiple frees
syncing disks... 3
done
Uptime: 12d10h34m24s
dumping to dev #ad/0x20001, offset 1575040
dump ata0: resetting devices .. done
(kgdb) bt
#0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:487
#1 0xc016926b in boot (howto=256) at /usr/src/sys/kern/kern_shutdown.c:316
#2 0xc0169690 in poweroff_wait (junk=0xc02d983f, howto=4) at
/usr/src/sys/kern/kern_shutdown.c:595
#3 0xc0164c87 in free (addr=0xc18f8880, type=0xc032d120) at
/usr/src/sys/kern/kern_malloc.c:385
#4 0xc018371e in cblock_free_cblocks (number=22) at
/usr/src/sys/kern/tty_subr.c:194
#5 0xc0183754 in clist_free_cblocks (clistp=0xc138fb38) at
/usr/src/sys/kern/tty_subr.c:208
#6 0xc017e89e in ttyclose (tp=0xc138fb00) at /usr/src/sys/kern/tty.c:247
#7 0xc02c67e9 in sioclose (dev=0xc11cc600, flag=7, mode=8192,
p=0xcbef8ee0) at /usr/src/sys/isa/sio.c:1669
#8 0xc01a2f78 in spec_close (ap=0xd56a7e44) at
/usr/src/sys/miscfs/specfs/spec_vnops.c:591
#9 0xc026a6d2 in ufsspec_close (ap=0xd56a7e44) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:1854
#10 0xc026ac99 in ufs_vnoperatespec (ap=0xd56a7e44) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:2394
#11 0xc019ea28 in vn_close (vp=0xd564a400, flags=7, cred=0xc0a36680,
p=0xcbef8ee0) at vnode_if.h:218
#12 0xc019f353 in vn_closefile (fp=0xc134a380, p=0xcbef8ee0) at
/usr/src/sys/kern/vfs_vnops.c:693
#13 0xc015ed2b in fdrop (fp=0xc134a380, p=0xcbef8ee0) at
/usr/src/sys/sys/file.h:218
#14 0xc015ec74 in closef (fp=0xc134a380, p=0xcbef8ee0) at
/usr/src/sys/kern/kern_descrip.c:1441
#15 0xc015ddab in close (p=0xcbef8ee0, uap=0xd56a7f80) at
/usr/src/sys/kern/kern_descrip.c:623
#16 0xc02b75f5 in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds =
-1078001617, tf_edi = -1077937232, tf_esi = -1077936924,
tf_ebp = -1077939744, tf_isp = -714440748, tf_ebx = -1077939748,
tf_edx = -1077936924, tf_ecx = 5, tf_eax = 6,
tf_trapno = 7, tf_err = 2, tf_eip = 672039156, tf_cs = 31, tf_eflags
= 647, tf_esp = -1077939804, tf_ss = 47})
at /usr/src/sys/i386/i386/trap.c:1175
#17 0xc02ab915 in Xint0x80_syscall ()
#18 0x804a9bf in ?? ()
#19 0x8049412 in ?? ()
#20 0x8048fca in ?? ()
(kgdb) bt full
#0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:487
error = 0
#1 0xc016926b in boot (howto=256) at /usr/src/sys/kern/kern_shutdown.c:316
howto = 256
#2 0xc0169690 in poweroff_wait (junk=0xc02d983f, howto=4) at
/usr/src/sys/kern/kern_shutdown.c:595
fmt = 0xc02d983f "free: multiple frees"
bootopt = 256
buf = "free: multiple frees", '\000' <repeats 235 times>
#3 0xc0164c87 in free (addr=0xc18f8880, type=0xc032d120) at
/usr/src/sys/kern/kern_malloc.c:385
kbp = (struct kmembuckets *) 0xc02d983f
kup = (struct kmemusage *) 0xc0a19b2c
freep = (struct freelist *) 0xc18f8880
size = 128
s = 6493208
ksp = (struct malloc_type *) 0x100
#4 0xc018371e in cblock_free_cblocks (number=22) at
/usr/src/sys/kern/tty_subr.c:194
number = 22
i = 4
#5 0xc0183754 in clist_free_cblocks (clistp=0xc138fb38) at
/usr/src/sys/kern/tty_subr.c:208
clistp = (struct clist *) 0xc138fb38
#6 0xc017e89e in ttyclose (tp=0xc138fb00) at /usr/src/sys/kern/tty.c:247
tp = (struct tty *) 0xc138fb00
s = 6493208
#7 0xc02c67e9 in sioclose (dev=0xc11cc600, flag=7, mode=8192,
p=0xcbef8ee0) at /usr/src/sys/isa/sio.c:1669
com = (struct com_s *) 0xc11ce800
mynor = 0
s = 0
tp = (struct tty *) 0xc138fb00
#8 0xc01a2f78 in spec_close (ap=0xd56a7e44) at
/usr/src/sys/miscfs/specfs/spec_vnops.c:591
vp = (struct vnode *) 0xd564a400
p = (struct proc *) 0xcbef8ee0
dev = 0xc11cc600
#9 0xc026a6d2 in ufsspec_close (ap=0xd56a7e44) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:1854
ap = (struct vop_close_args *) 0xd56a7e44
vp = (struct vnode *) 0x0
#10 0xc026ac99 in ufs_vnoperatespec (ap=0xd56a7e44) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:2394
ap = (struct vop_generic_args *) 0x0
#11 0xc019ea28 in vn_close (vp=0xd564a400, flags=7, cred=0xc0a36680,
p=0xcbef8ee0) at vnode_if.h:218
rc = -1053514880
a = {a_desc = 0xc0321280, a_vp = 0xd564a400, a_fflag = 7, a_cred =
0xc0a36680, a_p = 0xcbef8ee0}
vp = (struct vnode *) 0xd564a400
fflag = 0
cred = (struct ucred *) 0x0
---Type <return> to continue, or q <return> to quit---
p = (struct proc *) 0x0
vp = (struct vnode *) 0xd564a400
flags = 0
cred = (struct ucred *) 0x0
p = (struct proc *) 0x0
#12 0xc019f353 in vn_closefile (fp=0xc134a380, p=0xcbef8ee0) at
/usr/src/sys/kern/vfs_vnops.c:693
fp = (struct file *) 0x0
p = (struct proc *) 0x0
#13 0xc015ed2b in fdrop (fp=0xc134a380, p=0xcbef8ee0) at
/usr/src/sys/sys/file.h:218
fp = (struct file *) 0xc134a380
fp = (struct file *) 0xc134a380
lf = {l_start = -4502243104899727360, l_len =
-4629710226642173593, l_pid = 3, l_type = 32640, l_whence = -10902}
vp = (struct vnode *) 0x0
error = -873492768
#14 0xc015ec74 in closef (fp=0xc134a380, p=0xcbef8ee0) at
/usr/src/sys/kern/kern_descrip.c:1441
fp = (struct file *) 0xc134a380
p = (struct proc *) 0xcbef8ee0
vp = (struct vnode *) 0x0
lf = {l_start = -3751622871852515327, l_len =
-4605065915465564376, l_pid = -873492768, l_type = -23680, l_whence = -16076}
fdtol = (struct filedesc_to_leader *) 0x0
#15 0xc015ddab in close (p=0xcbef8ee0, uap=0xd56a7f80) at
/usr/src/sys/kern/kern_descrip.c:623
p = (struct proc *) 0xcbef8ee0
uap = (struct close_args *) 0x0
fdp = (struct filedesc *) 0xc138d000
fp = (struct file *) 0xc134a380
fd = 0
error = -1053241284
holdleaders = 0
#16 0xc02b75f5 in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds =
-1078001617, tf_edi = -1077937232, tf_esi = -1077936924,
tf_ebp = -1077939744, tf_isp = -714440748, tf_ebx = -1077939748,
tf_edx = -1077936924, tf_ecx = 5, tf_eax = 6,
tf_trapno = 7, tf_err = 2, tf_eip = 672039156, tf_cs = 31, tf_eflags
= 647, tf_esp = -1077939804, tf_ss = 47})
at /usr/src/sys/i386/i386/trap.c:1175
params = 0xbfbff1a8 ""
i = 0
callp = (struct sysent *) 0xc0327930
p = (struct proc *) 0xcbef8ee0
orig_tf_eflags = 647
sticks = 22993
error = 0
narg = 1
args = {0, -1077938412, 3, 0, 0, 0, 0, 0}
have_mplock = 1
code = 6
---Type <return> to continue, or q <return> to quit---
#17 0xc02ab915 in Xint0x80_syscall ()
No symbol table info available.
#18 0x804a9bf in ?? ()
No symbol table info available.
#19 0x8049412 in ?? ()
No symbol table info available.
#20 0x8048fca in ?? ()
No symbol table info available.
(kgdb)
And from a previous dump,
# gdb -k /usr/obj/usr/src/sys/gas/kernel.debug /var/crash/vmcore.0
GNU gdb 4.18 (FreeBSD)
Copyright 1998 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-unknown-freebsd"...Deprecated bfd_read
called at
/usr/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c
line 2627 in elfstab_build_psymtabs
Deprecated bfd_read called at
/usr/src/gnu/usr.bin/binutils/gdb/../../../../contrib/gdb/gdb/dbxread.c
line 933 in fill_symbuf
IdlePTD at physical address 0x003f5000
initial pcb at physical address 0x003493c0
panicstr: free: multiple frees
panic messages:
---
dmesg: kvm_read:
---
#0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:487
487 if (dumping++) {
(kgdb) bt
#0 dumpsys () at /usr/src/sys/kern/kern_shutdown.c:487
#1 0xc016926b in boot (howto=256) at /usr/src/sys/kern/kern_shutdown.c:316
#2 0xc0169690 in poweroff_wait (junk=0xc02d983f, howto=0) at
/usr/src/sys/kern/kern_shutdown.c:595
#3 0xc0164c87 in free (addr=0xc12b5480, type=0xc032d120) at
/usr/src/sys/kern/kern_malloc.c:385
#4 0xc018371e in cblock_free_cblocks (number=11) at
/usr/src/sys/kern/tty_subr.c:194
#5 0xc0183754 in clist_free_cblocks (clistp=0xc1351800) at
/usr/src/sys/kern/tty_subr.c:208
#6 0xc017e8a4 in ttyclose (tp=0xc1351800) at /usr/src/sys/kern/tty.c:248
#7 0xc02c67e9 in sioclose (dev=0xc11cc600, flag=7, mode=8192,
p=0xcbef8ee0) at /usr/src/sys/isa/sio.c:1669
#8 0xc01a2f78 in spec_close (ap=0xd56a7e44) at
/usr/src/sys/miscfs/specfs/spec_vnops.c:591
#9 0xc026a6d2 in ufsspec_close (ap=0xd56a7e44) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:1854
#10 0xc026ac99 in ufs_vnoperatespec (ap=0xd56a7e44) at
/usr/src/sys/ufs/ufs/ufs_vnops.c:2394
#11 0xc019ea28 in vn_close (vp=0xd564a400, flags=7, cred=0xc0a36680,
p=0xcbef8ee0) at vnode_if.h:218
#12 0xc019f353 in vn_closefile (fp=0xc134a140, p=0xcbef8ee0) at
/usr/src/sys/kern/vfs_vnops.c:693
#13 0xc015ed2b in fdrop (fp=0xc134a140, p=0xcbef8ee0) at
/usr/src/sys/sys/file.h:218
#14 0xc015ec74 in closef (fp=0xc134a140, p=0xcbef8ee0) at
/usr/src/sys/kern/kern_descrip.c:1441
#15 0xc015ddab in close (p=0xcbef8ee0, uap=0xd56a7f80) at
/usr/src/sys/kern/kern_descrip.c:623
#16 0xc02b75f5 in syscall2 (frame={tf_fs = 47, tf_es = 47, tf_ds =
-1078001617, tf_edi = -1077937232, tf_esi = -1077936924,
tf_ebp = -1077939744, tf_isp = -714440748, tf_ebx = -1077939748,
tf_edx = -1077936924, tf_ecx = 5, tf_eax = 6,
tf_trapno = 7, tf_err = 2, tf_eip = 672039156, tf_cs = 31, tf_eflags
= 647, tf_esp = -1077939804, tf_ss = 47})
at /usr/src/sys/i386/i386/trap.c:1175
#17 0xc02ab915 in Xint0x80_syscall ()
cannot read proc at 0xcbef8ee0
(kgdb)
Copyright (c) 1992-2004 The FreeBSD Project.
Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
The Regents of the University of California. All rights reserved.
FreeBSD 4.10-STABLE #5: Mon Sep 13 17:12:22 EDT 2004
gabor@pp-station.sentex.ca:/usr/obj/usr/src/sys/gas
Timecounter "i8254" frequency 1193182 Hz
Timecounter "TSC" frequency 2400410264 Hz
CPU: Intel(R) Celeron(R) CPU 2.40GHz (2400.41-MHz 686-class CPU)
Origin = "GenuineIntel" Id = 0xf29 Stepping = 9
Features=0xbfebfbff<FPU,VME,DE,PSE,TSC,MSR,PAE,MCE,CX8,APIC,SEP,MTRR,PGE,MCA,CMOV,PAT,PSE36,CLFLUSH,DTS,ACPI,MMX,FXSR,SSE,SSE2,SS,HTT,TM,PBE>
real memory = 267321344 (261056K bytes)
config> q
avail memory = 256217088 (250212K bytes)
Preloaded elf kernel "kernel" at 0xc03d6000.
Preloaded userconfig_script "/boot/kernel.conf" at 0xc03d609c.
Warning: Pentium 4 CPU: PSE disabled
Pentium Pro MTRR support enabled
md0: Malloc disk
Using $PIR table, 8 entries at 0xc00fdee0
npx0: <math processor> on motherboard
npx0: INT 16 interface
pcib0: <Host to PCI bridge> on motherboard
pci0: <PCI bus> on pcib0
agp0: <Intel 82865G (865G GMCH) SVGA controller> port 0xd000-0xd007 mem
0xfa000000-0xfa07ffff,0xf0000000-0xf7ffffff irq 15 at device 2.0 on pci0
agp0: detected 892k stolen memory
agp0: aperture size is 128M
uhci0: <Intel 82801EB (ICH5) USB controller USB-A> port 0xc000-0xc01f irq
15 at device 29.0 on pci0
usb0: <Intel 82801EB (ICH5) USB controller USB-A> on uhci0
usb0: USB revision 1.0
uhub0: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub0: 2 ports with 2 removable, self powered
uhid0: APC Back-UPS ES 500 FW:801.e5.D USB FW:e5, rev 1.10/1.06, addr 2,
iclass 3/0
uhci1: <Intel 82801EB (ICH5) USB controller USB-B> port 0xc400-0xc41f irq 5
at device 29.1 on pci0
usb1: <Intel 82801EB (ICH5) USB controller USB-B> on uhci1
usb1: USB revision 1.0
uhub1: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub1: 2 ports with 2 removable, self powered
uhci2: <Intel 82801EB (ICH5) USB controller USB-C> port 0xc800-0xc81f irq
10 at device 29.2 on pci0
usb2: <Intel 82801EB (ICH5) USB controller USB-C> on uhci2
usb2: USB revision 1.0
uhub2: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub2: 2 ports with 2 removable, self powered
uhci3: <Intel 82801EB (ICH5) USB controller USB-D> port 0xcc00-0xcc1f irq
15 at device 29.3 on pci0
usb3: <Intel 82801EB (ICH5) USB controller USB-D> on uhci3
usb3: USB revision 1.0
uhub3: Intel UHCI root hub, class 9/0, rev 1.00/1.00, addr 1
uhub3: 2 ports with 2 removable, self powered
pcib1: <Intel 82801BA/BAM (ICH2) Hub to PCI bridge> at device 30.0 on pci0
pci1: <PCI bus> on pcib1
rl0: <RealTek 8139 10/100BaseTX> port 0xa000-0xa0ff mem
0xf9000000-0xf90000ff irq 15 at device 4.0 on pci1
rl0: Ethernet address: 00:50:fc:ee:1b:34
miibus0: <MII bus> on rl0
rlphy0: <RealTek internal media interface> on miibus0
rlphy0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
puc0: <SmartLink 5634PCV SurfRider> port 0xa400-0xa407 irq 12 at device 5.0
on pci1
sio2: type 16550A
fxp0: <Intel 82801BA (D865) Pro/100 VE Ethernet> port 0xa800-0xa83f mem
0xf9001000-0xf9001fff irq 11 at device 8.0 on pci1
fxp0: Ethernet address 00:01:80:54:b5:65
inphy0: <i82562ET 10/100 media interface> on miibus1
inphy0: 10baseT, 10baseT-FDX, 100baseTX, 100baseTX-FDX, auto
puc1: <Lava Computers Quattro-PCI serial port> port
0xb000-0xb007,0xac00-0xac07 irq 10 at device 10.0 on pci1
sio3: type 16550A
sio4: type 16550A
puc2: <Lava Computers Quattro-PCI serial port> port
0xb800-0xb807,0xb400-0xb407 irq 10 at device 10.1 on pci1
sio5: type 16550A
sio6: type 16550A
isab0: <PCI to ISA bridge (vendor=8086 device=24d0)> at device 31.0 on pci0
isa0: <ISA bus> on isab0
atapci0: <Intel ICH5 ATA100 controller> port
0xf000-0xf00f,0-0x3,0-0x7,0-0x3,0-0x7 irq 0 at device 31.1 on pci0
ata0: at 0x1f0 irq 14 on atapci0
ata1: at 0x170 irq 15 on atapci0
ichsmb0: <Intel 82801EB (ICH5) SMBus controller> port 0x5000-0x501f irq 12
at device 31.3 on pci0
smbus0: <System Management Bus> on ichsmb0
smb0: <SMBus general purpose I/O> on smbus0
orm0: <Option ROM> at iomem 0xc0000-0xc9fff on isa0
pmtimer0 on isa0
fdc0: <NEC 72065B or clone> at port 0x3f0-0x3f5,0x3f7 irq 6 drq 2 on isa0
fdc0: FIFO enabled, 8 bytes threshold
atkbdc0: <Keyboard controller (i8042)> at port 0x60,0x64 on isa0
vga0: <Generic ISA VGA> at port 0x3c0-0x3df iomem 0xa0000-0xbffff on isa0
sc0: <System console> at flags 0x100 on isa0
sc0: VGA <16 virtual consoles, flags=0x300>
sio0 at port 0x3f8-0x3ff irq 4 flags 0x10 on isa0
sio0: type 16550A
sio1 at port 0x2f8-0x2ff irq 3 on isa0
sio1: type 16550A
ipfw2 initialized, divert enabled, rule-based forwarding enabled, default
to accept, logging limited to 100 packets/entry by default
IPsec: Initialized Security Association Processing.
ad0: 38166MB <ST340014A> [77545/16/63] at ata0-master UDMA100
Mounting root from ufs:/dev/ad0s1a
WARNING: / was not properly dismounted
--------------------------------------------------------------------
Mike Tancsa, tel +1 519 651 3400
Sentex Communications, mike@sentex.net
Providing Internet since 1994 www.sentex.net
Cambridge, Ontario Canada www.sentex.net/mike
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?6.1.2.0.0.20041014021404.04143d00>