Date: Mon, 29 Apr 2013 20:35:52 -0400 From: Glen Barber <gjb@FreeBSD.org> To: John Baldwin <jhb@freebsd.org> Cc: Ian FREISLICH <ianf@clue.co.za>, freebsd-current@freebsd.org Subject: Re: panic: in_pcblookup_local (?) Message-ID: <20130430003552.GC1588@glenbarber.us> In-Reply-To: <201304291224.06328.jhb@freebsd.org> References: <E1UW0K5-000P7H-36@clue.co.za> <20130428040256.GK1611@glenbarber.us> <201304291224.06328.jhb@freebsd.org>
next in thread | previous in thread | raw e-mail | index | archive | help
--7qSK/uQB79J36Y4o
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
Content-Transfer-Encoding: quoted-printable
On Mon, Apr 29, 2013 at 12:24:06PM -0400, John Baldwin wrote:
> On Sunday, April 28, 2013 12:02:56 am Glen Barber wrote:
> > On Sat, Apr 27, 2013 at 10:17:32AM +0200, Ian FREISLICH wrote:
> > > Hi
> > >=20
> > > I've been getting the following panic on recent current r249717.
> > > Sadly the crashdump is useless.
> > >=20
> >=20
> > I just saw similar panic on 10-CURRENT r249588.
> >=20
> > > Fatal trap 9: general protection fault while in kernel mode
> > > cpuid =3D 15; apic id =3D 0f
> > > instruction pointer =3D 0x20:0xffffffff80546fbc
> > > stack pointer =3D 0x28:0xffffff846b677770
> > > frame pointer =3D 0x28:0xffffff846b6777b0
> > > code segment =3D base 0x0, limit 0xfffff, type 0x1b
> > > =3D DPL 0, pres 1, long 1, def32 0, gran 1
> > > processor eflags =3D interrupt enabled, resume, IOPL =3D 0
> > > current process =3D 4361 (zabbix_agentd)
> >=20
> > Hmm.. This is interests me. In my case, cf-agent was the current
> > process.
> >=20
> > Backtrace of my panic follows. Any pointers on how to debug this
> > further would be appreciated.
> >=20
> > Glen
> >=20
> > Script started on Sat Apr 27 23:53:53 2013
> > root@orion:/usr/obj/usr/src/sys/ORION # kgdb ./kernel.debug=20
> /var/crash/vmcore.4
> > GNU gdb 6.1.1 [FreeBSD]
> > Copyright 2004 Free Software Foundation, Inc.
> > GDB is free software, covered by the GNU General Public License, and yo=
u are
> > welcome to change it and/or distribute copies of it under certain=20
> conditions.
> > Type "show copying" to see the conditions.
> > There is absolutely no warranty for GDB. Type "show warranty" for deta=
ils.
> > This GDB was configured as "amd64-marcel-freebsd"...
> >=20
> > Unread portion of the kernel message buffer:
> >=20
> >=20
> > Fatal trap 9: general protection fault while in kernel mode
> > cpuid =3D 1; apic id =3D 01
> > instruction pointer =3D 0x20:0xffffffff80736cec
> > stack pointer =3D 0x28:0xffffff81aad4e760
> > frame pointer =3D 0x28:0xffffff81aad4e7a0
> > code segment =3D base 0x0, limit 0xfffff, type 0x1b
> > =3D DPL 0, pres 1, long 1, def32 0, gran 1
> > processor eflags =3D interrupt enabled, resume, IOPL =3D 0
> > current process =3D 78664 (cf-agent)
> > trap number =3D 9
> > panic: general protection fault
> > cpuid =3D 1
> > KDB: stack backtrace:
> > #0 0xffffffff80642a56 at kdb_backtrace+0x66
> > #1 0xffffffff80606eeb at panic+0x13b
> > #2 0xffffffff808e3b10 at trap_fatal+0x290
> > #3 0xffffffff808e4331 at trap+0x241
> > #4 0xffffffff808cdbb3 at calltrap+0x8
> > #5 0xffffffff807371d8 at in_pcb_lport+0x128
> > #6 0xffffffff8073745a at in_pcbbind_setup+0x16a
> > #7 0xffffffff80737d8e at in_pcbconnect_setup+0x71e
> > #8 0xffffffff80737df9 at in_pcbconnect_mbuf+0x59
> > #9 0xffffffff807bf29f at udp_connect+0x11f
> > #10 0xffffffff80680615 at kern_connectat+0x275
> > #11 0xffffffff80680731 at sys_connect+0x41
> > #12 0xffffffff808e32cb at amd64_syscall+0x63b
> > #13 0xffffffff808cde97 at Xfast_syscall+0xf7
> > Uptime: 3d19h38m52s
> > (ada0:ahcich0:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 =
00 00
> > (ada0:ahcich0:0:0:0): CAM status: CCB request is in progress
> > (ada0:ahcich0:0:0:0): Error 5, Retries exhausted
> > (ada0:ahcich0:0:0:0): Synchronize cache failed
> > (ada1:ahcich1:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 =
00 00
> > (ada1:ahcich1:0:0:0): CAM status: CCB request is in progress
> > (ada1:ahcich1:0:0:0): Error 5, Retries exhausted
> > (ada1:ahcich1:0:0:0): Synchronize cache failed
> > (ada2:ahcich4:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 =
00 00
> > (ada2:ahcich4:0:0:0): CAM status: CCB request is in progress
> > (ada2:ahcich4:0:0:0): Error 5, Retries exhausted
> > (ada2:ahcich4:0:0:0): Synchronize cache failed
> > (ada3:ahcich5:0:0:0): FLUSHCACHE48. ACB: ea 00 00 00 00 40 00 00 00 00 =
00 00
> > (ada3:ahcich5:0:0:0): CAM status: CCB request is in progress
> > (ada3:ahcich5:0:0:0): Error 5, Retries exhausted
> > (ada3:ahcich5:0:0:0): Synchronize cache failed
> > Dumping 1014 out of 6049=20
> MB:..2%..12%..21%..32%..42%..51%..62%..71%..81%..92%
> >=20
> > Reading symbols from /boot/kernel/zfs.ko.symbols...done.
> > Loaded symbols for /boot/kernel/zfs.ko.symbols
> > Reading symbols from /boot/kernel/opensolaris.ko.symbols...done.
> > Loaded symbols for /boot/kernel/opensolaris.ko.symbols
> > #0 doadump (textdump=3D<value optimized out>) at pcpu.h:231
> > 231 __asm("movq %%gs:%1,%0" : "=3Dr" (td)
> > (kgdb) frame 6
> > #6 0xffffffff80736cec in in_pcblookup_local (pcbinfo=3D0xffffffff80dc9=
180,=20
> laddr=3D
> > {s_addr =3D 50374848}, lport=3D339, lookupflags=3D1,=20
> cred=3D0xfffffe016cdad100)
> > at /usr/src/sys/netinet/in_pcb.c:1438
> > 1438 LIST_FOREACH(phd, porthash, phd_hash) {
> > (kgdb) list *0xffffffff80736cec
> > 0xffffffff80736cec is in in_pcblookup_local=20
> (/usr/src/sys/netinet/in_pcb.c:1439).
> > 1434 * port hash list.
> > 1435 */
> > 1436 porthash =3D &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,
> > 1437 pcbinfo->ipi_porthashmask)];
> > 1438 LIST_FOREACH(phd, porthash, phd_hash) {
> > 1439 if (phd->phd_port =3D=3D lport)
> > 1440 break;
> > 1441 }
> > 1442 if (phd !=3D NULL) {
> > 1443 /*
>=20
> Can you see what 'phd' and 'porthash' are? If kgdb can't see them you can
> reconstruct what 'porthash' should be (you know the lport arg to this rou=
tine
> and can get the relevant fields from 'pcbinfo').
>=20
I'm not sure if the output included makes much sense. If I did not do
something correctly, please let me know.
Glen
Script started on Mon Apr 29 20:27:39 2013
root@orion:/usr/obj/usr/src/sys/ORION # kgdb ./kernel.debug /var/crash/vmco=
re.4
[...]
#0 doadump (textdump=3D<value optimized out>) at pcpu.h:231
231 __asm("movq %%gs:%1,%0" : "=3Dr" (td)
(kgdb) frame 6
#6 0xffffffff80736cec in in_pcblookup_local (pcbinfo=3D0xffffffff80dc9180,=
laddr=3D
{s_addr =3D 50374848}, lport=3D339, lookupflags=3D1, cred=3D0xfffffe0=
16cdad100)
at /usr/src/sys/netinet/in_pcb.c:1438
1438 LIST_FOREACH(phd, porthash, phd_hash) {
(kgdb) p *phd
Cannot access memory at address 0x9e17b100fffffe00
(kgdb) p *porthash
No symbol "porthash" in current context.
(kgdb) p &pcbinfo->ipi_porthashbase[INP_PCBPORTHASH(lport,pcbinfo->ipi_port=
hashmask)]
No symbol "INP_PCBPORTHASH" in current context.
(kgdb) p pcbinfo->ipi_porthashmask
$1 =3D 127
(kgdb) p *pcbinfo->ipi_porthashmask
Cannot access memory at address 0x7f
(kgdb) p *lport
Cannot access memory at address 0x153
(kgdb) p lport
$2 =3D 339
(kgdb) quit
root@orion:/usr/obj/usr/src/sys/ORION # ^D
Script done on Mon Apr 29 20:32:01 2013
--7qSK/uQB79J36Y4o
Content-Type: application/pgp-signature
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2.0.19 (FreeBSD)
iQEcBAEBCAAGBQJRfxHoAAoJEFJPDDeguUajKLUH/R7ytpI9yqiIaNR1pbSq6SfM
ORhfrluZ4apZRI8oCY2kIlx385eI1Jhca3u+o6cm+QP8kNATgNCeG3YEiiqrx+NR
D9u1YNASUUkksAyPcj7TYiDLCu1ZbcwHJnFUIY2g+eV9J25DsUtLf8CAJjgebUKO
3tZrjVFDmr+Svk/RKZr4+fxv1k6nXYqjNvVp3o4AXOJmcNI5/3Z3c/JUW7RwJ81v
MtJU7Gl7rqe6n819RSELFgxk0fKdfYEXCd0mL5pjnswC4yEubz22EruCkYSfaXMh
Iz9GVLqxLn+4yVp9rPw8eIt4ArbqG5H3q6FUw4YijcAXwlRNeaae0grP0Xfm3RI=
=irOW
-----END PGP SIGNATURE-----
--7qSK/uQB79J36Y4o--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20130430003552.GC1588>