Date: Wed, 19 Oct 2016 14:29:14 -0700 From: Gleb Smirnoff <glebius@FreeBSD.org> To: Mike Karels <mike@karels.net>, George Neville-Neil <gnn@freebsd.org> Cc: net@FreeBSD.org Subject: TCP route caching panic Message-ID: <20161019212914.GJ27748@FreeBSD.org>
next in thread | raw e-mail | index | archive | help
Hi!
I got this panic in a bhyve VM, which was just compiling stuff
and had two ssh sessions open. It has static network configuration,
one interfaces, one IP address, IPv4 only and it sees only hardware
address of the host. So it is minimal possible configuration. It
runs with INVARIANTS.
As you can see, cached route points to freed lle.
Unread portion of the kernel message buffer:
Fatal trap 9: general protection fault while in kernel mode
cpuid = 0; apic id = 00
instruction pointer = 0x20:0xffffffff805fff3a
stack pointer = 0x0:0xfffffe011c2c1bf0
frame pointer = 0x0:0xfffffe011c2c1de0
code segment = base 0x0, limit 0xfffff, type 0x1b
= DPL 0, pres 1, long 1, def32 0, gran 1
processor eflags = interrupt enabled, resume, IOPL = 0
current process = 12 (swi4: clock (0
doadump (textdump=250000) at /usr/src/head/sys/kern/kern_shutdown.c:298
298 dumptid = curthread->td_tid;
(kgdb) bt
#0 doadump (textdump=250000) at /usr/src/head/sys/kern/kern_shutdown.c:298
#1 0xffffffff80348265 in db_fncall_generic (addr=-2141161552, rv=0xfffffe011c2c1330, nargs=0, args=0xfffffe011c2c1340)
at /usr/src/head/sys/ddb/db_command.c:581
#2 0xffffffff80347914 in db_fncall (dummy1=-2194255637392, dummy2=false, dummy3=1, dummy4=0xfffffe011c2c1470 "\n")
at /usr/src/head/sys/ddb/db_command.c:629
#3 0xffffffff8034727e in db_command (last_cmdp=0xffffffff80e561b0 <db_last_command>, cmd_table=0x0, dopager=1)
at /usr/src/head/sys/ddb/db_command.c:453
#4 0xffffffff80346e19 in db_command_loop () at /usr/src/head/sys/ddb/db_command.c:506
#5 0xffffffff8034bfda in db_trap (type=9, code=0) at /usr/src/head/sys/ddb/db_main.c:248
#6 0xffffffff8066ad1f in kdb_trap (type=9, code=0, tf=0xfffffe011c2c1b30) at /usr/src/head/sys/kern/subr_kdb.c:654
#7 0xffffffff80a6d06d in trap_fatal (frame=0xfffffe011c2c1b30, eva=0) at /usr/src/head/sys/amd64/amd64/trap.c:832
#8 0xffffffff80a6c710 in trap (frame=0xfffffe011c2c1b30) at /usr/src/head/sys/amd64/amd64/trap.c:585
#9 0xffffffff80a6d78a in trap_check (frame=0xfffffe011c2c1b30) at /usr/src/head/sys/amd64/amd64/trap.c:638
#10 <signal handler called>
#11 0xffffffff805fff3a in __rw_wlock_hard (c=0xfffff800b1319d10, tid=18446735277663502336,
file=0xffffffff80b713d5 "/usr/src/head/sys/net/if_ethersubr.c", line=304) at /usr/src/head/sys/kern/kern_rwlock.c:831
#12 0xffffffff805ff9db in _rw_wlock_cookie (c=0xfffff800b1319d10, file=0xffffffff80b713d5 "/usr/src/head/sys/net/if_ethersubr.c", line=304)
at /usr/src/head/sys/kern/kern_rwlock.c:296
#13 0xffffffff807844a3 in ether_output (ifp=0xfffff80002d8f000, m=0xfffff800192aa500, dst=0xfffff8004894eac0, ro=0xfffff8004894eaa0)
at /usr/src/head/sys/net/if_ethersubr.c:304
#14 0xffffffff807ded62 in ip_output (m=0xfffff800192aa500, opt=0x0, ro=0xfffff8004894eaa0, flags=0, imo=0x0, inp=0xfffff8004894e910)
at /usr/src/head/sys/netinet/ip_output.c:661
#15 0xffffffff808c6bac in tcp_output (tp=0xfffff800b117d3d8) at /usr/src/head/sys/netinet/tcp_output.c:1432
#16 0xffffffff808d925a in tcp_timer_rexmt (xtp=0xfffff800b117d3d8) at /usr/src/head/sys/netinet/tcp_timer.c:856
#17 0xffffffff8062e0c4 in softclock_call_cc (c=0xfffff800b117d640, cc=0xffffffff8107f200 <cc_cpu>, direct=0)
at /usr/src/head/sys/kern/kern_timeout.c:729
#18 0xffffffff8062e78c in softclock (arg=0xffffffff8107f200 <cc_cpu>) at /usr/src/head/sys/kern/kern_timeout.c:867
#19 0xffffffff805b49e8 in intr_event_execute_handlers (p=0xfffff80002ca9a50, ie=0xfffff80002c94200) at /usr/src/head/sys/kern/kern_intr.c:1262
#20 0xffffffff805b56a7 in ithread_execute_handlers (p=0xfffff80002ca9a50, ie=0xfffff80002c94200) at /usr/src/head/sys/kern/kern_intr.c:1275
#21 0xffffffff805b54ec in ithread_loop (arg=0xfffff80002ca6f20) at /usr/src/head/sys/kern/kern_intr.c:1356
#22 0xffffffff805b056b in fork_exit (callout=0xffffffff805b53f0 <ithread_loop>, arg=0xfffff80002ca6f20, frame=0xfffffe011c2c2ac0)
at /usr/src/head/sys/kern/kern_fork.c:1038
#23 <signal handler called>
(kgdb) fr 13
#13 0xffffffff807844a3 in ether_output (ifp=0xfffff80002d8f000, m=0xfffff800192aa500, dst=0xfffff8004894eac0, ro=0xfffff8004894eaa0)
at /usr/src/head/sys/net/if_ethersubr.c:304
304 LLE_FREE(lle);
(kgdb) p lle
$1 = (struct llentry *) 0xfffff800b1319c00
(kgdb) fr 15
#15 0xffffffff808c6bac in tcp_output (tp=0xfffff800b117d3d8) at /usr/src/head/sys/netinet/tcp_output.c:1432
1432 error = ip_output(m, tp->t_inpcb->inp_options, &tp->t_inpcb->inp_route,
(kgdb) p tp->t_inpcb->inp_route
There is no member named inp_route.
(kgdb) p tp->t_inpcb->inp_rtu.inpu_route
$4 = {ro_rt = 0xfffff80003df95b0, ro_lle = 0xfffff800b1319c00, ro_prepend = 0x0, ro_plen = 0, ro_flags = 258, ro_mtu = 0, spare = 0, ro_dst = {
sa_len = 16 '\020', sa_family = 2 '\002', sa_data = "\000\000\n\006\006\b\000\000\000\000\000\000\000"}}
(kgdb) p tp->t_inpcb->inp_rtu.inpu_route->ro_lle
$5 = (struct llentry *) 0xfffff800b1319c00
(kgdb) p *tp->t_inpcb->inp_rtu.inpu_route->ro_lle
$6 = {lle_next = {le_next = 0xfffff8000388ea00, le_prev = 0xfffff8001706e600}, r_l3addr = {addr4 = {s_addr = 54176312}, addr6 = {__u6_addr = {
__u6_addr8 = "8\252:\003\000\370\377\377\001\001\200\000\336\300\255\336", __u6_addr16 = {43576, 826, 63488, 65535, 257, 128, 49374,
57005}, __u6_addr32 = {54176312, 4294965248, 8388865, 3735929054}}}},
r_linkdata = "\000\000\000\000\000\000\000\000\350]d\002\000\376\377\377\336\300\255\336\336\300\255\336", r_hdrlen = 222 '\336',
spare0 = "\300\255\336", r_flags = 49374, r_skip_req = 57005, lle_tbl = 0xfffff800033ee000, lle_head = 0xffff0000002d6bbd,
lle_free = 0xdeadc0dedeadc0de, la_hold = 0xdeadc0dedeadc0de, la_numheld = 393712128, la_expire = 0, la_flags = 21632, la_asked = 45489,
la_preempt = 63488, ln_state = -1, ln_router = 21664, ln_ntick = -1, lle_remtime = -1, lle_hittime = 0, lle_refcnt = 0, ll_addr = 0x0,
lle_chain = {le_next = 0x0, le_prev = 0xfffff8001960b300}, lle_timer = {c_links = {le = {le_next = 0xfffff8001715f378, le_prev = 0x0}, sle = {
sle_next = 0xfffff8001715f378}, tqe = {tqe_next = 0xfffff8001715f378, tqe_prev = 0x0}}, c_time = -8793120203584, c_precision = 0,
c_arg = 0xfffff800b1319cd0, c_func = 0x0, c_lock = 0xfffff800b1319ce0, c_flags = 0, c_iflags = 0, c_cpu = 0}, lle_lock = {lock_object = {
lo_name = 0xfffff800b1319cf0 "", lo_flags = 0, lo_data = 0, lo_witness = 0xdeadc0dedeadc0de}, rw_lock = 16045693110842147038}, req_mtx = {
lock_object = {lo_name = 0xdeadc0dedeadc0de <error: Cannot access memory at address 0xdeadc0dedeadc0de>, lo_flags = 3735929054,
lo_data = 3735929054, lo_witness = 0xdeadc0dedeadc0de}, mtx_lock = 16045693110842147038}}
(kgdb)
--
Totus tuus, Glebius.
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20161019212914.GJ27748>