Date: Wed, 19 Jan 2011 17:51:59 +0300 From: Sergey Kandaurov <pluknet@gmail.com> To: John Baldwin <jhb@freebsd.org> Cc: freebsd-current@freebsd.org, Robert Watson <rwatson@freebsd.org> Subject: Re: uma_zalloc_arg: zone "256" with non-sleepable exclusive rw ifnet_rw @ /usr/src/sys/net/if.c:414 Message-ID: <AANLkTimj_6VR2gwb4oxhiFYvV%2B8gdrU2-V8b%2BE5_6%2BXT@mail.gmail.com> In-Reply-To: <201101181404.26768.jhb@freebsd.org> References: <AANLkTi=dZ4vOSx82Wp%2BqAwmT97mucqwtf6dDu4pnOCR2@mail.gmail.com> <201101180954.46903.jhb@freebsd.org> <AANLkTi=DtUa5%2BfhOrOW_N11rn6WVp8fHZvL%2BOE6n-S6c@mail.gmail.com> <201101181404.26768.jhb@freebsd.org>
next in thread | previous in thread | raw e-mail | index | archive | help
On 18 January 2011 22:04, John Baldwin <jhb@freebsd.org> wrote:
> On Tuesday, January 18, 2011 1:22:24 pm Sergey Kandaurov wrote:
>> On 18 January 2011 17:54, John Baldwin <jhb@freebsd.org> wrote:
>> > On Monday, January 17, 2011 12:55:26 pm Sergey Kandaurov wrote:
>> >> Hi,
>> >>
>> >> I see this "malloc with non-sleepable" on current during boot.
>> >> It's strange that I don't see it if I boot via pxe/nfs.
>> >>
>> >> if_alloc() calls ifindex_alloc_locked() under IFNET_WLOCK() which
>> >> might call if_grow().
>> >> Looks like a regression from r196553.
>> >
>> > I'm guessing that ifindex_alloc() should drop the lock and retry the
>> > allocation after calling if_grow()? =A0This compiles, but I haven't bo=
oted
> it
>> > yet:
>>
>> vnet_if_init() calls if_grow() without lock.
>
> So it does. :( =A0I've added locking to the sysinit to handle this:
Seems a bit more work there. The vnet_if_init() sysinit cannot use ifnet
locking since it runs before another sysinit initialized those ifnet locks.
On the other side it's safe to call if_grow() from vnet_if_init() w/o locki=
ng:
db> bt
Tracing pid 0 tid 100000 td 0xffffffff80ccffc0
_sx_xlock_hard() at _sx_xlock_hard+0xa0
_sx_xlock() at _sx_xlock+0xbb
vnet_if_init() at vnet_if_init+0x4a
mi_startup() at mi_startup+0x77
btext() at btext+0x2c
After said certain locking changes I came to the next problem.
A modified ifindex_alloc_locked() function doesn't update/increment V_if_in=
dex
in successful if_grow() case, so it ends up in quick memory exhaustion:
bce0: <Broadcom NetXtreme II BCM5709 1000Base-T (C0)> mem
0x92000000-0x93ffffff irq 28 at device 0.0 on pci4
panic: kmem_malloc(-2147483648): kmem_map too small: 1110716416 total alloc=
ated
cpuid =3D 0
KDB: enter: panic
[ thread pid 0 tid 100000 ]
Stopped at kdb_enter+0x3d: movq $0,0x700140(%rip)
db> bt
Tracing pid 0 tid 100000 td 0xffffffff80ccffc0
kdb_enter() at kdb_enter+0x3d
panic() at panic+0x180
kmem_malloc() at kmem_malloc+0x25d
uma_large_malloc() at uma_large_malloc+0x4a
malloc() at malloc+0x15d
if_grow() at if_grow+0x98
if_alloc() at if_alloc+0xd8
bce_attach() at bce_attach+0x18de
[...]
> show malloc
ifnet 2 1048578K 25
So I added V_if_index increment into if_grow() itself.
At least it boots now :)
There are 18 visible interfaces in the test system, so it looks appropriate
minus one extra elm.
ifnet 19 16420K - 36 128,256,512,1024,2048,4096
db> show all ifnet
bce0 ifp=3D0xfffffe0019426000
igb0 ifp=3D0xfffffe0019425800
igb1 ifp=3D0xfffffe0019425000
igb2 ifp=3D0xfffffe0016caf800
igb3 ifp=3D0xfffffe00195d3800
igb4 ifp=3D0xfffffe00195d2800
igb5 ifp=3D0xfffffe00195d2000
usbus0 ifp=3D0xfffffe0019426800
usbus1 ifp=3D0xfffffe00195d3000
usbus2 ifp=3D0xfffffe0019902800
usbus3 ifp=3D0xfffffe0019902000
usbus4 ifp=3D0xfffffe0019901800
usbus5 ifp=3D0xfffffe0019901000
usbus6 ifp=3D0xfffffe0019900800
usbus7 ifp=3D0xfffffe0019900000
igb6 ifp=3D0xfffffe0019d99000
igb7 ifp=3D0xfffffe0019d98800
lo0 ifp=3D0xfffffe0019d98000
%%%
Index: sys/net/if.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- sys/net/if.c (revision 217503)
+++ sys/net/if.c (working copy)
@@ -266,6 +266,7 @@ ifindex_alloc_locked(u_short *idxp)
IFNET_WLOCK_ASSERT();
+retry:
/*
* Try to find an empty slot below V_if_index. If we fail, take th=
e
* next slot.
@@ -278,10 +279,11 @@ ifindex_alloc_locked(u_short *idxp)
/* Catch if_index overflow. */
if (idx < 1)
return (ENOSPC);
- if (idx > V_if_index)
- V_if_index =3D idx;
- if (V_if_index >=3D V_if_indexlim)
+ if (idx > V_if_index) {
if_grow();
+ goto retry;
+ }
+ V_if_index =3D idx;
*idxp =3D idx;
return (0);
}
@@ -351,7 +353,10 @@ vnet_if_init(const void *unused __unused)
TAILQ_INIT(&V_ifnet);
TAILQ_INIT(&V_ifg_head);
+ IFNET_LOCK_INIT(); /* XXX */
+ IFNET_WLOCK();
if_grow(); /* create initial table */
+ IFNET_WUNLOCK();
vnet_if_clone_init();
}
VNET_SYSINIT(vnet_if_init, SI_SUB_INIT_IF, SI_ORDER_FIRST, vnet_if_init,
@@ -362,7 +367,6 @@ static void
if_init(void *dummy __unused)
{
- IFNET_LOCK_INIT();
if_clone_init();
}
SYSINIT(interfaces, SI_SUB_INIT_IF, SI_ORDER_SECOND, if_init, NULL);
@@ -385,16 +389,26 @@ VNET_SYSUNINIT(vnet_if_uninit, SI_SUB_INIT_IF, SI_
static void
if_grow(void)
{
+ int oldlim;
u_int n;
struct ifindex_entry *e;
- V_if_indexlim <<=3D 1;
- n =3D V_if_indexlim * sizeof(*e);
+ IFNET_WLOCK_ASSERT();
+ oldlim =3D V_if_indexlim;
+ IFNET_WUNLOCK();
+ n =3D (oldlim << 1) * sizeof(*e);
e =3D malloc(n, M_IFNET, M_WAITOK | M_ZERO);
+ IFNET_WLOCK();
+ if (V_if_indexlim !=3D oldlim) {
+ free(e, M_IFNET);
+ return;
+ }
if (V_ifindex_table !=3D NULL) {
memcpy((caddr_t)e, (caddr_t)V_ifindex_table, n/2);
free((caddr_t)V_ifindex_table, M_IFNET);
}
+ V_if_indexlim <<=3D 1;
+ V_if_index++;
V_ifindex_table =3D e;
}
%%%
--=20
wbr,
pluknet
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?AANLkTimj_6VR2gwb4oxhiFYvV%2B8gdrU2-V8b%2BE5_6%2BXT>