Date: Tue, 16 Mar 2010 08:45:28 +0200 From: Mikolaj Golub <to.my.trociny@gmail.com> To: freebsd-net@freebsd.org Cc: Kostik Belousov <kostikbel@gmail.com> Subject: Re: kmem leakage on tun/tap device removal Message-ID: <86634wdat3.fsf@zhuzha.ua1> References: <mailpost.1267356711.2478384.62287.mailing.freebsd.net@FreeBSD.cs.nctu.edu.tw>
next in thread | previous in thread | raw e-mail | index | archive | help
--=-=-=
Content-Type: text/plain; charset=gb2312
Content-Transfer-Encoding: quoted-printable
On Feb 28, 1:30 pm, to.my.trociny@gmail.com (Mikolaj Golub) wrote:
> But I have faced with another issue (not related to your patch, as it is
> observed with unpatched kernel too). When I try to run concurrently two
> create/destroy scripts with the same interface the system panics:
>=20
> Unread portion of the kernel message buffer:
> panic: Bad link elm 0xc5f1a800 next->prev !=3D elm
> cpuid =3D 2
> KDB: enter: panic
> exclusive sleep mutex if_clone lock (if_clone lock) r =3D 0 (0xc0da1cf0) =
locked @ /usr/src/sys/net/if_clone.c:248
> exclusive sleep mutex if_clone lock (if_clone lock) r =3D 0 (0xc0da1cf0) =
locked @ /usr/src/sys/net/if_clone.c:248
> exclusive sx so_rcv_sx (so_rcv_sx) r =3D 0 (0xc6cd3560) locked @ /usr/src=
/sys/kern/uipc_sockbuf.c:148
> exclusive sx so_rcv_sx (so_rcv_sx) r =3D 0 (0xc6b4dbd0) locked @ /usr/src=
/sys/kern/uipc_sockbuf.c:148
> Physical memory: 2019 MB
> Dumping 160 MB: 145 129 113 97 81 65 49 33 17 1
>=20
> #0 doadump () at pcpu.h:246
> 246 __asm __volatile("movl %%fs:0,%0" : "=3Dr" (td));
> (kgdb) bt
> #0 doadump () at pcpu.h:246
> #1 0xc04e8bb9 in db_fncall (dummy1=3D-1064515926, dummy2=3D0, dummy3=3D-=
1, dummy4=3D0xe83f4834 "HH?=A8=A8")
> at /usr/src/sys/ddb/db_command.c:548
> #2 0xc04e8fef in db_command (last_cmdp=3D0xc0de14dc, cmd_table=3D0x0, do=
pager=3D0)
> at /usr/src/sys/ddb/db_command.c:445
> #3 0xc04e90a4 in db_command_script (command=3D0xc0de2404 "call doadump")
> at /usr/src/sys/ddb/db_command.c:516
> #4 0xc04ed1d0 in db_script_exec (scriptname=3D0xe83f4940 "kdb.enter.pani=
c", warnifnotfound=3DVariable "warnifnotfound" is not available.
> )
> at /usr/src/sys/ddb/db_script.c:302
> #5 0xc04ed2b7 in db_script_kdbenter (eventname=3D0xc0ca1948 "panic") at =
/usr/src/sys/ddb/db_script.c:324
> #6 0xc04eaf98 in db_trap (type=3D3, code=3D0) at /usr/src/sys/ddb/db_mai=
n.c:228
> #7 0xc08cc526 in kdb_trap (type=3D3, code=3D0, tf=3D0xe83f4a7c) at /usr/=
src/sys/kern/subr_kdb.c:535
> #8 0xc0bdd38b in trap (frame=3D0xe83f4a7c) at /usr/src/sys/i386/i386/tra=
p.c:690
> #9 0xc0bbef1b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
> #10 0xc08cc6aa in kdb_enter (why=3D0xc0ca1948 "panic", msg=3D0xc0ca1948 "=
panic") at cpufunc.h:71
> #11 0xc089d716 in panic (fmt=3D0xc0c3c80c "Bad link elm %p next->prev !=
=3D elm")
> at /usr/src/sys/kern/kern_shutdown.c:562
> #12 0xc094e7fb in if_clone_destroyif (ifc=3D0xc0da1cc0, ifp=3D0xc5f1a800)=
at /usr/src/sys/net/if_clone.c:249
> #13 0xc094eb52 in if_clone_destroy (name=3D0xc664ac20 "tun0") at /usr/src=
/sys/net/if_clone.c:227
> #14 0xc094c8a6 in ifioctl (so=3D0xc6e0a9a8, cmd=3D2149607801, data=3D0xc6=
64ac20 "tun0", td=3D0xc66c0d80)
> at /usr/src/sys/net/if.c:2412
> #15 0xc08e8b25 in soo_ioctl (fp=3D0xc6d46af0, cmd=3D2149607801, data=3D0x=
c664ac20, active_cred=3D0xc5f62280,
> td=3D0xc66c0d80) at /usr/src/sys/kern/sys_socket.c:212
> #16 0xc08e31bd in kern_ioctl (td=3D0xc66c0d80, fd=3D3, com=3D2149607801, =
data=3D0xc664ac20 "tun0") at file.h:262
> #17 0xc08e3344 in ioctl (td=3D0xc66c0d80, uap=3D0xe83f4cf8) at /usr/src/s=
ys/kern/sys_generic.c:678
> #18 0xc0bdca33 in syscall (frame=3D0xe83f4d38) at /usr/src/sys/i386/i386/=
trap.c:1078
> #19 0xc0bbefb0 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception=
.s:261
> #20 0x00000033 in ?? ()
> Previous frame inner to this frame (corrupt stack?)
> (kgdb) fr 12
> #12 0xc094e7fb in if_clone_destroyif (ifc=3D0xc0da1cc0, ifp=3D0xc5f1a800)=
at /usr/src/sys/net/if_clone.c:249
> 249 IFC_IFLIST_REMOVE(ifc, ifp);
> (kgdb) list
> 244 * switch to the vnet context of the target vnet.
> 245 */
> 246 CURVNET_SET_QUIET(ifp->if_vnet);
> 247
> 248 IF_CLONE_LOCK(ifc);
> 249 IFC_IFLIST_REMOVE(ifc, ifp);
> 250 IF_CLONE_UNLOCK(ifc);
> 251
> 252 if_delgroup(ifp, ifc->ifc_name);
> 253
>=20
Actually, this issue has already been reported (kern/116837, see the bottom=
of
the discussion) and there was a patch provided by Takahiro Kurosawa [check
that ifp is on ifc->ifc_iflist before calling IFC_IFLIST_REMOVE(ifc, ifp)].
Although he mentioned that another race was still possible. I have tried the
patch and yes it makes the situation much better: the box did not crush when
running two "ifconfig tun0 create/destroy" scripts concurrently, but when I
tried 8 concurrent processes :-) it crashed after a couple minutes in anoth=
er
place:
(kgdb) bt
#0 doadump () at pcpu.h:246
#1 0xc04ec379 in db_fncall (dummy1=3D1, dummy2=3D0, dummy3=3D-1056947200, =
dummy4=3D0xe86848e4 "")
at /usr/src/sys/ddb/db_command.c:548
#2 0xc04ec771 in db_command (last_cmdp=3D0xc0e04d1c, cmd_table=3D0x0, dopa=
ger=3D1)
at /usr/src/sys/ddb/db_command.c:445
#3 0xc04ec8ca in db_command_loop () at /usr/src/sys/ddb/db_command.c:498
#4 0xc04ee76d in db_trap (type=3D12, code=3D0) at /usr/src/sys/ddb/db_main=
.c:229
#5 0xc08d7d06 in kdb_trap (type=3D12, code=3D0, tf=3D0xe8684ad0) at /usr/s=
rc/sys/kern/subr_kdb.c:535
#6 0xc0bea66f in trap_fatal (frame=3D0xe8684ad0, eva=3D3735929054) at /usr=
/src/sys/i386/i386/trap.c:929
#7 0xc0beaf90 in trap (frame=3D0xe8684ad0) at /usr/src/sys/i386/i386/trap.=
c:328
#8 0xc0bccd7b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
#9 0xc094bfa6 in strcmp (s1=3D0xc663686b "vmnet", s2=3D0xdeadc0de <Address=
0xdeadc0de out of bounds>)
at /usr/src/sys/libkern/strcmp.c:45
#10 0xc095a9c2 in if_clone_destroy (name=3D0xc5f7d840 "tun0") at /usr/src/s=
ys/net/if_clone.c:209
#11 0xc09584d6 in ifioctl (so=3D0xc721a80c, cmd=3D2149607801, data=3D0xc5f7=
d840 "tun0", td=3D0xc731fb90)
at /usr/src/sys/net/if.c:2486
#12 0xc08f4615 in soo_ioctl (fp=3D0xc5f1ca80, cmd=3D2149607801, data=3D0xc5=
f7d840, active_cred=3D0xc5ed4180,=20
td=3D0xc731fb90) at /usr/src/sys/kern/sys_socket.c:212
#13 0xc08eec8d in kern_ioctl (td=3D0xc731fb90, fd=3D3, com=3D2149607801, da=
ta=3D0xc5f7d840 "tun0") at file.h:262
#14 0xc08eee14 in ioctl (td=3D0xc731fb90, uap=3D0xe8684cf8) at /usr/src/sys=
/kern/sys_generic.c:678
#15 0xc0beab40 in syscall (frame=3D0xe8684d38) at /usr/src/sys/i386/i386/tr=
ap.c:1111
#16 0xc0bcce10 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s=
:261
#17 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 10
#10 0xc095a9c2 in if_clone_destroy (name=3D0xc5f7d840 "tun0") at /usr/src/s=
ys/net/if_clone.c:209
209 if (strcmp(ifc->ifc_name, ifp->if_dname) =3D=3D 0) {
(kgdb) list
204 return (ENXIO);
205=20=20=20=20=20
206 /* Find the cloner for this interface */
207 IF_CLONERS_LOCK();
208 LIST_FOREACH(ifc, &V_if_cloners, ifc_list) {
209 if (strcmp(ifc->ifc_name, ifp->if_dname) =3D=3D 0) {
210 break;
211 }
212 }
213 #ifdef VIMAGE
(kgdb) p ifc->ifc_name
$1 =3D 0xc663686b "vmnet"
(kgdb) p ifp->if_dname
$2 =3D 0xdeadc0de <Address 0xdeadc0de out of bounds>
(kgdb)=20
May be we can use ifunit_ref() instead of ifunit() like in the patch below =
to
avoid this race (the patch also includes Takahiro Kurosawa's patch from
kern/116837)?
I was running 32 "ifconfig tun0 create/destroy" on the patched kernel throu=
gh
all the night and did not manage to crash the system.
--=20
Mikolaj Golub
--=-=-=
Content-Type: text/x-patch
Content-Disposition: inline; filename=if_clone.c.if_clone_destroy.patch
--- sys/net/if_clone.c.orig 2010-02-28 16:39:30.000000000 +0200
+++ sys/net/if_clone.c 2010-03-15 23:46:56.000000000 +0200
@@ -196,10 +196,11 @@ if_clone_createif(struct if_clone *ifc,
int
if_clone_destroy(const char *name)
{
+ int err;
struct if_clone *ifc;
struct ifnet *ifp;
- ifp = ifunit(name);
+ ifp = ifunit_ref(name);
if (ifp == NULL)
return (ENXIO);
@@ -221,10 +222,14 @@ if_clone_destroy(const char *name)
}
#endif
IF_CLONERS_UNLOCK();
- if (ifc == NULL)
+ if (ifc == NULL) {
+ if_rele(ifp);
return (EINVAL);
+ }
- return (if_clone_destroyif(ifc, ifp));
+ err = if_clone_destroyif(ifc, ifp);
+ if_rele(ifp);
+ return err;
}
/*
@@ -234,6 +239,7 @@ int
if_clone_destroyif(struct if_clone *ifc, struct ifnet *ifp)
{
int err;
+ struct ifnet *tmp;
if (ifc->ifc_destroy == NULL)
return(EOPNOTSUPP);
@@ -246,8 +252,15 @@ if_clone_destroyif(struct if_clone *ifc,
CURVNET_SET_QUIET(ifp->if_vnet);
IF_CLONE_LOCK(ifc);
- IFC_IFLIST_REMOVE(ifc, ifp);
+ LIST_FOREACH(tmp, &ifc->ifc_iflist, if_clones) {
+ if (tmp == ifp) {
+ IFC_IFLIST_REMOVE(ifc, ifp);
+ break;
+ }
+ }
IF_CLONE_UNLOCK(ifc);
+ if (tmp == NULL)
+ return (ENXIO); /* ifp is not on the list. */
if_delgroup(ifp, ifc->ifc_name);
--=-=-=--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?86634wdat3.fsf>