Date: Sun, 24 Jul 2005 10:19:11 -0400 From: Edwin <edwin@verolan.com> To: freebsd-hackers@freebsd.org Cc: Edwin <edwin@verolan.com>, Max Laier <max@love2party.net> Subject: Re: help w/panic under heavy load - 5.4 Message-ID: <20050724141911.GA26240@asx01.verolan.com> In-Reply-To: <200507241546.08255.max@love2party.net> References: <20050719034215.GB20752@asx01.verolan.com> <200507240027.54127.max@love2party.net> <20050724023845.GA15209@asx01.verolan.com> <200507241546.08255.max@love2party.net>
next in thread | previous in thread | raw e-mail | index | archive | help
New kernel: ident D1-0723 (same as D1-0722 - but w/ IPFIREWALL* options removed)
same traces asked for previously.
Thanks again,
/Edwin
kgdb kernel.debug /usr/local/STORAGE/crash/vmcore.1
[GDB will not be able to debug user-mode threads: /usr/lib/libthread_db.so: Undefined symbol "ps_pglobal_lookup"]
GNU gdb 6.1.1 [FreeBSD]
Copyright 2004 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-marcel-freebsd".
#0 doadump () at pcpu.h:159
159 __asm __volatile("movl %%fs:0,%0" : "=r" (td));
(kgdb) where
#0 doadump () at pcpu.h:159
#1 0xc0460ef6 in db_fncall (dummy1=0, dummy2=0, dummy3=43, dummy4=0xc76bf9f4 "(úkÇ")
at /usr/src/sys/ddb/db_command.c:531
#2 0xc0460d04 in db_command (last_cmdp=0xc08be624, cmd_table=0x0, aux_cmd_tablep=0xc083e324,
aux_cmd_tablep_end=0xc083e340) at /usr/src/sys/ddb/db_command.c:349
#3 0xc0460dcc in db_command_loop () at /usr/src/sys/ddb/db_command.c:455
#4 0xc0462951 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:221
#5 0xc06277f2 in kdb_trap (type=3, code=0, tf=0xc76bfb30) at /usr/src/sys/kern/subr_kdb.c:468
#6 0xc07ad874 in trap (frame=
{tf_fs = -949288936, tf_es = -1067319280, tf_ds = -1065287664, tf_edi = 1, tf_esi = -1065233792, tf_ebp = -949224592, tf_isp = -949224612, tf_ebx = -949224548, tf_edx = 0, tf_ecx = -1060921344, tf_eax = 18, tf_trapno = 3, tf_err = 0, tf_eip = -1067289229, tf_cs = -1065287672, tf_eflags = 658, tf_esp = -949224560, tf_ss = -1067377425})
at /usr/src/sys/i386/i386/trap.c:584
#7 0xc079deaa in calltrap () at /usr/src/sys/i386/i386/exception.s:140
#8 0xc76b0018 in ?? ()
#9 0xc0620010 in sched_runnable () at /usr/src/sys/kern/sched_4bsd.c:641
#10 0xc0611cef in panic (fmt=0xc081d280 "m_copym, offset > size of mbuf chain")
at /usr/src/sys/kern/kern_shutdown.c:550
#11 0xc064172c in m_copym (m=0x0, off0=1500, len=1480, wait=1) at /usr/src/sys/kern/uipc_mbuf.c:385
#12 0xc0692b74 in ip_fragment (ip=0xc12f000e, m_frag=0xc76bfc6c, mtu=-1056775680, if_hwassist_flags=0, sw_csum=1)
at /usr/src/sys/netinet/ip_output.c:967
#13 0xc068f6e9 in ip_fastforward (m=0xc12e2300) at /usr/src/sys/netinet/ip_fastfwd.c:572
#14 0xc0672759 in ether_demux (ifp=0xc0f90000, m=0xc12e2300) at /usr/src/sys/net/if_ethersubr.c:770
#15 0xc06724f5 in ether_input (ifp=0xc0f90000, m=0xc12e2300) at /usr/src/sys/net/if_ethersubr.c:631
#16 0xc070a9e7 in sis_rxeof (sc=0xc0f90000) at /usr/src/sys/pci/if_sis.c:1636
#17 0xc070ae6f in sis_intr (arg=0xc0f90000) at /usr/src/sys/pci/if_sis.c:1841
#18 0xc0600130 in ithread_loop (arg=0xc0ec6880) at /usr/src/sys/kern/kern_intr.c:547
#19 0xc05ff5a4 in fork_exit (callout=0xc060000c <ithread_loop>, arg=0xc0ec6880, frame=0xc76bfd48)
at /usr/src/sys/kern/kern_fork.c:791
#20 0xc079df0c in fork_trampoline () at /usr/src/sys/i386/i386/exception.s:209
(kgdb) f 13
#13 0xc068f6e9 in ip_fastforward (m=0xc12e2300) at /usr/src/sys/netinet/ip_fastfwd.c:572
572 if (ip_fragment(ip, &m, mtu, ifp->if_hwassist,
(kgdb) l
567 m->m_pkthdr.csum_flags |= CSUM_IP;
568 /*
569 * ip_fragment expects ip_len and ip_off in host byte
570 * order but returns all packets in network byte order
571 */
572 if (ip_fragment(ip, &m, mtu, ifp->if_hwassist,
573 (~ifp->if_hwassist & CSUM_DELAY_IP))) {
574 goto drop;
575 }
576 KASSERT(m != NULL, ("null mbuf and no error"));
(kgdb) i loc
ip = (struct ip *) 0xc12f000e
m0 = (struct mbuf *) 0xc12f000e
ro = {ro_rt = 0xc11ee420, ro_dst = {sa_len = 16 '\020', sa_family = 2 '\002',
sa_data = "\000\000ˬ\002\005\000\000\000\000\000\000\000"}}
dst = (struct sockaddr_in *) 0xc76bfc3c
ia = (struct in_ifaddr *) 0x0
ifa = (struct ifaddr *) 0x0
ifp = (struct ifnet *) 0xc0f91800
odest = {s_addr = 84060352}
dest = {s_addr = 84060352}
sum = 0
ip_len = 0
error = 84060352
hlen = -1057417216
mtu = 0
__func__ = "ip_fastforward"
(kgdb) p *ip
$1 = {ip_hl = 5, ip_v = 4, ip_tos = 0 '\0', ip_len = 10240, ip_id = 33436, ip_off = 0, ip_ttl = 64 '@',
ip_p = 17 '\021', ip_sum = 59733, ip_src = {s_addr = 67479744}, ip_dst = {s_addr = 84060352}}
(kgdb) p *m
$2 = {m_hdr = {mh_next = 0x0, mh_nextpkt = 0x0, mh_data = 0xc12f000e "E", mh_len = 40, mh_flags = 3,
mh_type = 1}, M_dat = {MH = {MH_pkthdr = {rcvif = 0xc0f90000, len = 40, header = 0x0, csum_flags = 769,
csum_data = 0, tags = {slh_first = 0x0}}, MH_dat = {MH_ext = {ext_buf = 0xc12f0000 "", ext_free = 0,
ext_args = 0x0, ext_size = 2048, ref_cnt = 0x0, ext_type = 3},
MH_databuf = "\000\000/Á\000\000\000\000\000\000\000\000\000\b\000\000\000\000\000\000\003", '\0' <repeats 186 times>}},
M_databuf = "\000\000ùÀ(\000\000\000\000\000\000\000\001\003", '\0' <repeats 12 times>, "/Á\000\000\000\000\000\000\000\000\000\b\000\000\000\000\000\000\003", '\0' <repeats 186 times>}}
(kgdb) p *ro.ro_rt
$3 = {rt_nodes = {{rn_mklist = 0x0, rn_parent = 0xc11ee438, rn_bit = -1, rn_bmask = 0 '\0', rn_flags = 4 '\004',
rn_u = {rn_leaf = {rn_Key = 0xc1069480 "\020\002", rn_Mask = 0x0, rn_Dupedkey = 0x0}, rn_node = {
rn_Off = -1056533376, rn_L = 0x0, rn_R = 0x0}}}, {rn_mklist = 0x0, rn_parent = 0xc11ee120, rn_bit = 61,
rn_bmask = 4 '\004', rn_flags = 4 '\004', rn_u = {rn_leaf = {rn_Key = 0x7 <Address 0x7 out of bounds>,
rn_Mask = 0xc11ef000 "à\201üÀ8ä\036ÁÇÿ", rn_Dupedkey = 0xc11ee420}, rn_node = {rn_Off = 7,
rn_L = 0xc11ef000, rn_R = 0xc11ee420}}}}, rt_gateway = 0xc1069490, rt_flags = 132101,
rt_ifp = 0xc0f91800, rt_ifa = 0xc104f900, rt_rmx = {rmx_mtu = 1500, rmx_expire = 334076357, rmx_pksent = 7238},
rt_refcnt = 1, rt_genmask = 0x0, rt_llinfo = 0xc105d040 "`Ð\005Á", rt_gwroute = 0x0, rt_parent = 0xc11ef000,
rt_mtx = {mtx_object = {lo_class = 0xc0876a9c, lo_name = 0xc08213a5 "rtentry", lo_type = 0xc08213a5 "rtentry",
lo_flags = 4390912, lo_list = {tqe_next = 0x0, tqe_prev = 0x0}, lo_witness = 0x0}, mtx_lock = 4,
mtx_recurse = 0}}
(kgdb) p *ifp
$4 = {if_softc = 0xc0f91800, if_link = {tqe_next = 0xc0f90000, tqe_prev = 0xc08e1244},
if_xname = "sis0", '\0' <repeats 11 times>, if_dname = 0xc0f2ecec "sis", if_dunit = 0, if_addrhead = {
tqh_first = 0xc0ec0000, tqh_last = 0xc1038460}, if_klist = {kl_lock = 0xc08dae00, kl_list = {
slh_first = 0x0}}, if_pcount = 0, if_carp = 0x0, if_bpf = 0x0, if_index = 1, if_timer = 5, if_nvlans = 0,
if_flags = 34883, if_capabilities = 72, if_capenable = 72, if_linkmib = 0x0, if_linkmiblen = 0, if_data = {
ifi_type = 6 '\006', ifi_physical = 0 '\0', ifi_addrlen = 6 '\006', ifi_hdrlen = 18 '\022',
ifi_link_state = 2 '\002', ifi_recvquota = 0 '\0', ifi_xmitquota = 0 '\0', ifi_datalen = 80 'P',
ifi_mtu = 1500, ifi_metric = 0, ifi_baudrate = 10000000, ifi_ipackets = 40, ifi_ierrors = 0,
ifi_opackets = 7276, ifi_oerrors = 0, ifi_collisions = 0, ifi_ibytes = 4728, ifi_obytes = 394534,
ifi_imcasts = 30, ifi_omcasts = 24, ifi_iqdrops = 0, ifi_noproto = 0, ifi_hwassist = 0, ifi_epoch = 0,
ifi_lastchange = {tv_sec = 0, tv_usec = 0}}, if_multiaddrs = {tqh_first = 0xc0f94620, tqh_last = 0xc0fab5e0},
if_amcount = 0, if_output = 0xc0671b04 <ether_output>, if_input = 0xc0672298 <ether_input>,
if_start = 0xc070b0f0 <sis_start>, if_ioctl = 0xc070be5c <sis_ioctl>, if_watchdog = 0xc070bfe4 <sis_watchdog>,
if_init = 0xc070b440 <sis_init>, if_resolvemulti = 0xc0672b48 <ether_resolvemulti>, if_spare1 = 0x0,
if_spare2 = 0x0, if_spare3 = 0x0, if_spare_flags1 = 0, if_spare_flags2 = 0, if_snd = {ifq_head = 0x0,
ifq_tail = 0x0, ifq_len = 0, ifq_maxlen = 127, ifq_drops = 0, ifq_mtx = {mtx_object = {lo_class = 0xc0876a9c,
lo_name = 0xc0f9180c "sis0", lo_type = 0xc082053b "if send queue", lo_flags = 196608, lo_list = {
tqe_next = 0x0, tqe_prev = 0x0}, lo_witness = 0x0}, mtx_lock = 4, mtx_recurse = 0}, ifq_drv_head = 0x0,
ifq_drv_tail = 0x0, ifq_drv_len = 0, ifq_drv_maxlen = 127, altq_type = 0, altq_flags = 1, altq_disc = 0x0,
altq_ifp = 0xc0f91800, altq_enqueue = 0, altq_dequeue = 0, altq_request = 0, altq_clfier = 0x0,
altq_classify = 0, altq_tbr = 0x0, altq_cdnr = 0x0}, if_broadcastaddr = 0xc07d2ae0 "ÿÿÿÿÿÿ", lltables = 0x0,
if_label = 0x0, if_prefixhead = {tqh_first = 0x0, tqh_last = 0xc0f91968}, if_afdata = {0x0 <repeats 28 times>,
0xc0faaad0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, if_afdata_initialized = 1, if_afdata_mtx = {
mtx_object = {lo_class = 0xc0876a9c, lo_name = 0xc082052b "if_afdata", lo_type = 0xc082052b "if_afdata",
lo_flags = 196608, lo_list = {tqe_next = 0x0, tqe_prev = 0x0}, lo_witness = 0x0}, mtx_lock = 4,
mtx_recurse = 0}, if_starttask = {ta_link = {stqe_next = 0x0}, ta_pending = 0, ta_priority = 0,
ta_func = 0xc0670ec0 <if_start_deferred>, ta_context = 0xc0f91800}}
(kgdb) f 12
#12 0xc0692b74 in ip_fragment (ip=0xc12f000e, m_frag=0xc76bfc6c, mtu=-1056775680, if_hwassist_flags=0, sw_csum=1)
at /usr/src/sys/netinet/ip_output.c:967
967 m->m_next = m_copy(m0, off, len);
(kgdb) i loc
mhip = (struct ip *) 0xc102e240
m = (struct mbuf *) 0xc102e200
mhlen = 20
error = 0
hlen = 20
len = 1480
off = 1500
m0 = (struct mbuf *) 0xc12e2300
firstlen = 1480
mnext = (struct mbuf **) 0xc12e2304
nfrags = 1
(kgdb) exit
Undefined command: "exit". Try "help".
(kgdb) quit
mbsd05#
Max Laier (max@love2party.net) wrote:
>
> > If you feel that it's a ipfw/ipfil issue - I can easily take IPFIREWALL*
> > options out of the kernel and build a new one - just give me about 15
> > minutes.
>
> Yes please and make sure it isn't loaded as a module either.
>
> --
> /"\ Best regards, | mlaier@freebsd.org
> \ / Max Laier | ICQ #67774661
> X http://pf4freebsd.love2party.net/ | mlaier@EFnet
> / \ ASCII Ribbon Campaign | Against HTML Mail and News
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20050724141911.GA26240>