Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 2 May 2010 17:27:53 GMT
From:      Mikolaj Golub <to.my.trociny@gmail.com>
To:        freebsd-gnats-submit@FreeBSD.org
Subject:   misc/146250: Races on interface alias removal
Message-ID:  <201005021727.o42HRr40025514@www.freebsd.org>
Resent-Message-ID: <201005021730.o42HUEFE091393@freefall.freebsd.org>

next in thread | raw e-mail | index | archive | help

>Number:         146250
>Category:       misc
>Synopsis:       Races on interface alias removal
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-bugs
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          sw-bug
>Submitter-Id:   current-users
>Arrival-Date:   Sun May 02 17:30:13 UTC 2010
>Closed-Date:
>Last-Modified:
>Originator:     Mikolaj Golub
>Release:        8.0-STABLE
>Organization:
>Environment:
FreeBSD zhuzha.ua1 8.0-STABLE FreeBSD 8.0-STABLE #11: Tue Apr 20 13:20:40 EEST 2010     root@zhuzha.ua1:/us
/usr/src/sys/DEBUG  i386

>Description:
When the same alias is removed by two threads concurrently reces are possible, which lead to kernel panic. So far I have seen 4 types of panics:

1. A crash in in_control() on removing ia->ia_ifa from ifp->if_addrhead list, because there is no check if the address is still in the list before removing:

panic: Bad link elm 0xcd2f3b00 prev->next != elm

#0  doadump () at pcpu.h:246
#1  0xc04ec829 in db_fncall (dummy1=-1064461270, dummy2=0, dummy3=-1, dummy4=0xe9a737fc "\0208&#9558;&#1048;")
    at /usr/src/sys/ddb/db_command.c:548
#2  0xc04ecc5f in db_command (last_cmdp=0xc0e0ab9c, cmd_table=0x0, dopager=0)
    at /usr/src/sys/ddb/db_command.c:445
#3  0xc04ecd14 in db_command_script (command=0xc0e0bac4 "call doadump") at /usr/src/sys/ddb/db_command.c:516
#4  0xc04f0e50 in db_script_exec (scriptname=0xe9a73908 "kdb.enter.panic", warnifnotfound=Variable "warnifnotfound" is not available.
)
    at /usr/src/sys/ddb/db_script.c:302
#5  0xc04f0f37 in db_script_kdbenter (eventname=0xc0cc760a "panic") at /usr/src/sys/ddb/db_script.c:324
#6  0xc04eec18 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:228
#7  0xc08d9aa6 in kdb_trap (type=3, code=0, tf=0xe9a73a44) at /usr/src/sys/kern/subr_kdb.c:535
#8  0xc0befbeb in trap (frame=0xe9a73a44) at /usr/src/sys/i386/i386/trap.c:690
#9  0xc0bd130b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
#10 0xc08d9c2a in kdb_enter (why=0xc0cc760a "panic", msg=0xc0cc760a "panic") at cpufunc.h:71
#11 0xc08a95b6 in panic (fmt=0xc0c61bc0 "Bad link elm %p prev->next != elm")
    at /usr/src/sys/kern/kern_shutdown.c:562
#12 0xc09ba87f in in_control (so=0xcdbd519c, cmd=2149607705, data=0xcd3db120 "fxp0", ifp=0xc5b94c00, 
    td=0xc92ddb90) at /usr/src/sys/netinet/in.c:604
#13 0xc095d400 in ifioctl (so=0xcdbd519c, cmd=2149607705, data=0xcd3db120 "fxp0", td=0xc92ddb90)
    at /usr/src/sys/net/if.c:2516
#14 0xc08f69d5 in soo_ioctl (fp=0xcdc90af0, cmd=2149607705, data=0xcd3db120, active_cred=0xc9d78400, 
    td=0xc92ddb90) at /usr/src/sys/kern/sys_socket.c:212
#15 0xc08f0a2d in kern_ioctl (td=0xc92ddb90, fd=3, com=2149607705, data=0xcd3db120 "fxp0") at file.h:262
#16 0xc08f0bb4 in ioctl (td=0xc92ddb90, uap=0xe9a73cf8) at /usr/src/sys/kern/sys_generic.c:678
#17 0xc0bef320 in syscall (frame=0xe9a73d38) at /usr/src/sys/i386/i386/trap.c:1111
#18 0xc0bd13a0 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:261
#19 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 12
#12 0xc09ba87f in in_control (so=0xcdbd519c, cmd=2149607705, data=0xcd3db120 "fxp0", ifp=0xc5b94c00, 
    td=0xc92ddb90) at /usr/src/sys/netinet/in.c:604
604             TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link);
(kgdb) list
599             default:
600                     panic("in_control: unsupported ioctl");
601             }
602
603             IF_ADDR_LOCK(ifp);
604             TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link);
605             IF_ADDR_UNLOCK(ifp);
606             ifa_free(&ia->ia_ifa);                          /* if_addrhead */
607
608             IN_IFADDR_WLOCK();

2. A crash in in_lltable_prefix_free.

(kgdb) bt
#0  doadump () at pcpu.h:246
#1  0xc04ec829 in db_fncall (dummy1=1, dummy2=0, dummy3=-1056922880, dummy4=0xe8636760 "")
    at /usr/src/sys/ddb/db_command.c:548
#2  0xc04ecc21 in db_command (last_cmdp=0xc0e0ac1c, cmd_table=0x0, dopager=1)
    at /usr/src/sys/ddb/db_command.c:445
#3  0xc04ecd7a in db_command_loop () at /usr/src/sys/ddb/db_command.c:498
#4  0xc04eec1d in db_trap (type=12, code=0) at /usr/src/sys/ddb/db_main.c:229
#5  0xc08d9aa6 in kdb_trap (type=12, code=0, tf=0xe863694c) at /usr/src/sys/kern/subr_kdb.c:535
#6  0xc0beeedf in trap_fatal (frame=0xe863694c, eva=420) at /usr/src/sys/i386/i386/trap.c:929
#7  0xc0bef800 in trap (frame=0xe863694c) at /usr/src/sys/i386/i386/trap.c:328
#8  0xc0bd139b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
#9  0xc08a6a8b in _rw_wlock_hard (rw=0xc79e1508, tid=3334964384, file=0xc0ce01e4 "/usr/src/sys/netinet/in.c", 
    line=1370) at /usr/src/sys/kern/kern_rwlock.c:677
#10 0xc08a75d6 in _rw_wlock (rw=0xc79e1508, file=0xc0ce01e4 "/usr/src/sys/netinet/in.c", line=1370)
    at /usr/src/sys/kern/kern_rwlock.c:237
#11 0xc09bb17e in in_lltable_prefix_free (llt=0xc5dabc00, prefix=0xe8636a94, mask=0xe8636a84)
    at /usr/src/sys/netinet/in.c:1370
#12 0xc09631d1 in lltable_prefix_free (af=2, prefix=0xe8636a94, mask=0xe8636a84)
    at /usr/src/sys/net/if_llatbl.c:217
#13 0xc09b8d77 in in_ifscrub (ifp=0xc5b94c00, ia=0xc6ec0500) at /usr/src/sys/netinet/in.c:1197
#14 0xc09ba6dc in in_control (so=0xc79d2338, cmd=2149607705, data=0xc629b0c0 "fxp0", ifp=0xc5b94c00, 
    td=0xc6c784a0) at /usr/src/sys/netinet/in.c:586
#15 0xc095d400 in ifioctl (so=0xc79d2338, cmd=2149607705, data=0xc629b0c0 "fxp0", td=0xc6c784a0)
    at /usr/src/sys/net/if.c:2516
#16 0xc08f69d5 in soo_ioctl (fp=0xc6304738, cmd=2149607705, data=0xc629b0c0, active_cred=0xc79d8d80, 
    td=0xc6c784a0) at /usr/src/sys/kern/sys_socket.c:212
#17 0xc08f0a2d in kern_ioctl (td=0xc6c784a0, fd=3, com=2149607705, data=0xc629b0c0 "fxp0") at file.h:262
#18 0xc08f0bb4 in ioctl (td=0xc6c784a0, uap=0xe8636cf8) at /usr/src/sys/kern/sys_generic.c:678
#19 0xc0bef3b0 in syscall (frame=0xe8636d38) at /usr/src/sys/i386/i386/trap.c:1111
#20 0xc0bd1430 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:261
#21 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 11
#11 0xc09bb17e in in_lltable_prefix_free (llt=0xc5dabc00, prefix=0xe8636a94, mask=0xe8636a84)
    at /usr/src/sys/netinet/in.c:1370
1370                                    LLE_WLOCK(lle);
(kgdb) list
1365                    LIST_FOREACH_SAFE(lle, &llt->lle_head[i], lle_next, next) {
1366
1367                            if (IN_ARE_MASKED_ADDR_EQUAL((struct sockaddr_in *)L3_ADDR(lle), 
1368                                                         pfx, msk)) {
1369                                    callout_drain(&lle->la_timer);
1370                                    LLE_WLOCK(lle);
1371                                    llentry_free(lle);
1372                            }
1373                    }
1374            }
(kgdb) p *lle
$1 = {lle_next = {le_next = 0xdeadc0de, le_prev = 0xdeadc0de}, lle_lock = {lock_object = {
      lo_name = 0xdeadc0de <Address 0xdeadc0de out of bounds>, lo_flags = 3735929054, lo_data = 3735929054, 
      lo_witness = 0xdeadc0de}, rw_lock = 3735929054}, lle_tbl = 0xdeadc0de, lle_head = 0xdeadc0de, 
  la_hold = 0xdeadc0de, la_expire = -559038242, la_flags = 49374, la_asked = 57005, la_preempt = 49374, 
  ln_byhint = 57005, ln_state = -16162, ln_router = 57005, ln_ntick = -559038242, lle_refcnt = -559038242, 
  ll_addr = {mac_aligned = 16045693110842147038, mac16 = {49374, 57005, 49374}}, lle_timer = {ln_timer_ch = {
      c_links = {sle = {sle_next = 0xdeadc0de}, tqe = {tqe_next = 0xdeadc0de, tqe_prev = 0xdeadc0de}}, 
      c_time = -559038242, c_arg = 0xdeadc0de, c_func = 0xdeadc0de, c_lock = 0xdeadc0de, 
      c_flags = -559038242, c_cpu = -559038242}, la_timer = {c_links = {sle = {sle_next = 0xdeadc0de}, tqe = {
          tqe_next = 0xdeadc0de, tqe_prev = 0xdeadc0de}}, c_time = -559038242, c_arg = 0xdeadc0de, 
      c_func = 0xdeadc0de, c_lock = 0xdeadc0de, c_flags = -559038242, c_cpu = -559038242}}}
(kgdb) fr 12
#12 0xc09631d1 in lltable_prefix_free (af=2, prefix=0xe8636a94, mask=0xe8636a84)
    at /usr/src/sys/net/if_llatbl.c:217
217                     llt->llt_prefix_free(llt, prefix, mask);
(kgdb) list
212             LLTABLE_RLOCK();
213             SLIST_FOREACH(llt, &V_lltables, llt_link) {
214                     if (llt->llt_af != af)
215                             continue;
216
217                     llt->llt_prefix_free(llt, prefix, mask);
218             }
219             LLTABLE_RUNLOCK();
220     }
221

lltable is RLOCKed while the entries are deleted from the table. When
callout_drain() is run by one thread other thread has time to destroy lle.

3. A crash in sysctl_iflist(), processing ifa, which is destroyed by
other thread.

(kgdb) bt
#0  doadump () at pcpu.h:246
#1  0xc08a92fe in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:416
#2  0xc08a95d2 in panic (fmt=Variable "fmt" is not available.
) at /usr/src/sys/kern/kern_shutdown.c:579
#3  0xc0beef23 in trap_fatal (frame=0xe8456a64, eva=3735929054) at /usr/src/sys/i386/i386/trap.c:938
#4  0xc0bef113 in trap_pfault (frame=0xe8456a64, usermode=0, eva=3735929054)
    at /usr/src/sys/i386/i386/trap.c:851
#5  0xc0befb05 in trap (frame=0xe8456a64) at /usr/src/sys/i386/i386/trap.c:533
#6  0xc0bd139b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
#7  0xc096f85e in rt_msg2 (type=12, rtinfo=0xe8456b08, cp=0x0, w=0xe8456b38) at /usr/src/sys/net/rtsock.c:1022
#8  0xc096ff7b in sysctl_rtsock (oidp=0xc0dcb900, arg1=0xe8456c18, arg2=4, req=0xe8456ba4)
    at /usr/src/sys/net/rtsock.c:1408
#9  0xc08b4598 in sysctl_root (oidp=Variable "oidp" is not available.
) at /usr/src/sys/kern/kern_sysctl.c:1418
#10 0xc08b475c in userland_sysctl (td=0xc636b250, name=0xe8456c10, namelen=6, old=0x0, oldlenp=0xbfbfe048, 
    inkernel=0, new=0x0, newlen=0, retval=0xe8456c70, flags=0) at /usr/src/sys/kern/kern_sysctl.c:1522
#11 0xc08b4b14 in __sysctl (td=0xc636b250, uap=0xe8456cf8) at /usr/src/sys/kern/kern_sysctl.c:1448
#12 0xc0bef3b0 in syscall (frame=0xe8456d38) at /usr/src/sys/i386/i386/trap.c:1111
#13 0xc0bd1430 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:261
#14 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 8
#8  0xc096ff7b in sysctl_rtsock (oidp=0xc0dcb900, arg1=0xe8456c18, arg2=4, req=0xe8456ba4)
    at /usr/src/sys/net/rtsock.c:1408
1408                            len = rt_msg2(RTM_NEWADDR, &info, NULL, w);
(kgdb) list 1370,1410
1370    static int
1371    sysctl_iflist(int af, struct walkarg *w)
1372    {
1373            struct ifnet *ifp;
1374            struct ifaddr *ifa;
1375            struct rt_addrinfo info;
1376            int len, error = 0;
1377
1378            bzero((caddr_t)&info, sizeof(info));
1379            IFNET_RLOCK();
1380            TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
1381                    if (w->w_arg && w->w_arg != ifp->if_index)
1382                            continue;
1383                    ifa = ifp->if_addr;
1384                    info.rti_info[RTAX_IFP] = ifa->ifa_addr;
1385                    len = rt_msg2(RTM_IFINFO, &info, NULL, w);
1386                    info.rti_info[RTAX_IFP] = NULL;
1387                    if (w->w_req && w->w_tmem) {
1388                            struct if_msghdr *ifm;
1389
1390                            ifm = (struct if_msghdr *)w->w_tmem;
1391                            ifm->ifm_index = ifp->if_index;
1392                            ifm->ifm_flags = ifp->if_flags | ifp->if_drv_flags;
1393                            ifm->ifm_data = ifp->if_data;
1394                            ifm->ifm_addrs = info.rti_addrs;
1395                            error = SYSCTL_OUT(w->w_req,(caddr_t)ifm, len);
1396                            if (error)
1397                                    goto done;
1398                    }
1399                    while ((ifa = TAILQ_NEXT(ifa, ifa_link)) != NULL) {
1400                            if (af && af != ifa->ifa_addr->sa_family)
1401                                    continue;
1402                            if (prison_if(w->w_req->td->td_ucred,
1403                                ifa->ifa_addr) != 0)
1404                                    continue;
1405                            info.rti_info[RTAX_IFA] = ifa->ifa_addr;
1406                            info.rti_info[RTAX_NETMASK] = ifa->ifa_netmask;
1407                            info.rti_info[RTAX_BRD] = ifa->ifa_dstaddr;
1408                            len = rt_msg2(RTM_NEWADDR, &info, NULL, w);
1409                            if (w->w_req && w->w_tmem) {
1410                                    struct ifa_msghdr *ifam;
(kgdb) fr 7  
#7  0xc096f85e in rt_msg2 (type=12, rtinfo=0xe8456b08, cp=0x0, w=0xe8456b38) at /usr/src/sys/net/rtsock.c:1022
1022                    rtinfo->rti_addrs |= (1 << i);
(kgdb) p *rtinfo
$2 = {rti_addrs = 4, rti_info = {0x0, 0x0, 0xdeadc0de, 0x0, 0x0, 0xdeadc0de, 0x0, 0xdeadc0de}, rti_flags = 0, 
  rti_ifa = 0x0, rti_ifp = 0x0}

4. A crush in in_ifinit() on removing ia from the list because there is no check if the element is still in the list.

panic: Bad link elm 0xc876ea00 prev->next != elm

#0  doadump () at pcpu.h:246
#1  0xc04ec829 in db_fncall (dummy1=-1064461270, dummy2=0, dummy3=-1, dummy4=0xe880d784 "\230&#1074;\200&#1061;")
    at /usr/src/sys/ddb/db_command.c:548
#2  0xc04ecc5f in db_command (last_cmdp=0xc0e0ac9c, cmd_table=0x0, dopager=0)
    at /usr/src/sys/ddb/db_command.c:445
#3  0xc04ecd14 in db_command_script (command=0xc0e0bbc4 "call doadump") at /usr/src/sys/ddb/db_command.c:516
#4  0xc04f0e50 in db_script_exec (scriptname=0xe880d890 "kdb.enter.panic", warnifnotfound=Variable "warnifnotfound" is not available.
)
    at /usr/src/sys/ddb/db_script.c:302
#5  0xc04f0f37 in db_script_kdbenter (eventname=0xc0cc770a "panic") at /usr/src/sys/ddb/db_script.c:324
#6  0xc04eec18 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_main.c:228
#7  0xc08d9aa6 in kdb_trap (type=3, code=0, tf=0xe880d9cc) at /usr/src/sys/kern/subr_kdb.c:535
#8  0xc0befcfb in trap (frame=0xe880d9cc) at /usr/src/sys/i386/i386/trap.c:690
#9  0xc0bd141b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
#10 0xc08d9c2a in kdb_enter (why=0xc0cc770a "panic", msg=0xc0cc770a "panic") at cpufunc.h:71
#11 0xc08a95b6 in panic (fmt=0xc0c61cc0 "Bad link elm %p prev->next != elm")
    at /usr/src/sys/kern/kern_shutdown.c:562
#12 0xc09b8efc in in_ifinit (ifp=0xc5b94c00, ia=0xc876ea00, sin=0xc185fcf6, scrub=0)
    at /usr/src/sys/netinet/in.c:844
#13 0xc09ba58b in in_control (so=0xc7b13ce0, cmd=2151704858, data=0xc7841bc0 "fxp0", ifp=0xc5b94c00, 
    td=0xc818db90) at /usr/src/sys/netinet/in.c:564
#14 0xc095d400 in ifioctl (so=0xc7b13ce0, cmd=2151704858, data=0xc7841bc0 "fxp0", td=0xc818db90)
    at /usr/src/sys/net/if.c:2516
#15 0xc08f69d5 in soo_ioctl (fp=0xc70a84d0, cmd=2151704858, data=0xc7841bc0, active_cred=0xc78dc400, 
    td=0xc818db90) at /usr/src/sys/kern/sys_socket.c:212
#16 0xc08f0a2d in kern_ioctl (td=0xc818db90, fd=3, com=2151704858, data=0xc7841bc0 "fxp0") at file.h:262
#17 0xc08f0bb4 in ioctl (td=0xc818db90, uap=0xe880dcf8) at /usr/src/sys/kern/sys_generic.c:678
#18 0xc0bef430 in syscall (frame=0xe880dd38) at /usr/src/sys/i386/i386/trap.c:1111
#19 0xc0bd14b0 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:261
#20 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 12
#12 0xc09b8efc in in_ifinit (ifp=0xc5b94c00, ia=0xc876ea00, sin=0xc185fcf6, scrub=0)
    at /usr/src/sys/netinet/in.c:844
844                     LIST_REMOVE(ia, ia_hash);
(kgdb) list in_ifinit
832      * and routing table entry.
833      */
834     static int
835     in_ifinit(struct ifnet *ifp, struct in_ifaddr *ia, struct sockaddr_in *sin,
836         int scrub)
837     {
838             register u_long i = ntohl(sin->sin_addr.s_addr);
839             struct sockaddr_in oldaddr;
840             int s = splimp(), flags = RTF_UP, error = 0;
841     
(kgdb) 
842             oldaddr = ia->ia_addr;
843             if (oldaddr.sin_family == AF_INET)
844                     LIST_REMOVE(ia, ia_hash);
845             ia->ia_addr = *sin;
846             if (ia->ia_addr.sin_family == AF_INET) {
847                     IN_IFADDR_WLOCK();
848                     LIST_INSERT_HEAD(INADDR_HASH(ia->ia_addr.sin_addr.s_addr),
849                         ia, ia_hash);
850                     IN_IFADDR_WUNLOCK();
851             }


5. Another crash in in_control() on referencing dead memory.

#0  doadump () at pcpu.h:246
#1  0xc04ec829 in db_fncall (dummy1=1, dummy2=0, dummy3=-1056922624, dummy4=0xe847c890 "")
    at /usr/src/sys/ddb/db_command.c:548
#2  0xc04ecc21 in db_command (last_cmdp=0xc0e0ad1c, cmd_table=0x0, dopager=1)
    at /usr/src/sys/ddb/db_command.c:445
#3  0xc04ecd7a in db_command_loop () at /usr/src/sys/ddb/db_command.c:498
#4  0xc04eec1d in db_trap (type=12, code=0) at /usr/src/sys/ddb/db_main.c:229
#5  0xc08d9aa6 in kdb_trap (type=12, code=0, tf=0xe847ca7c) at /usr/src/sys/kern/subr_kdb.c:535
#6  0xc0beefbf in trap_fatal (frame=0xe847ca7c, eva=3735929146) at /usr/src/sys/i386/i386/trap.c:929
#7  0xc0bef8e0 in trap (frame=0xe847ca7c) at /usr/src/sys/i386/i386/trap.c:328
#8  0xc0bd147b in calltrap () at /usr/src/sys/i386/i386/exception.s:165
#9  0xc09b9c24 in in_control (so=0xc6e29670, cmd=2149607705, data=0xc6246ba0 "fxp0", ifp=0xc5b94c00, 
    td=0xc6a59940) at /usr/src/sys/netinet/in.c:331
#10 0xc095d400 in ifioctl (so=0xc6e29670, cmd=2149607705, data=0xc6246ba0 "fxp0", td=0xc6a59940)
    at /usr/src/sys/net/if.c:2516
#11 0xc08f69d5 in soo_ioctl (fp=0xc6374700, cmd=2149607705, data=0xc6246ba0, active_cred=0xc7131280, 
    td=0xc6a59940) at /usr/src/sys/kern/sys_socket.c:212
#12 0xc08f0a2d in kern_ioctl (td=0xc6a59940, fd=3, com=2149607705, data=0xc6246ba0 "fxp0") at file.h:262
#13 0xc08f0bb4 in ioctl (td=0xc6a59940, uap=0xe847ccf8) at /usr/src/sys/kern/sys_generic.c:678
#14 0xc0bef490 in syscall (frame=0xe847cd38) at /usr/src/sys/i386/i386/trap.c:1111
#15 0xc0bd1510 in Xint0x80_syscall () at /usr/src/sys/i386/i386/exception.s:261
#16 0x00000033 in ?? ()
Previous frame inner to this frame (corrupt stack?)
(kgdb) fr 9
#9  0xc09b9c24 in in_control (so=0xc6e29670, cmd=2149607705, data=0xc6246ba0 "fxp0", ifp=0xc5b94c00, 
    td=0xc6a59940) at /usr/src/sys/netinet/in.c:331
331                     if (iap->ia_ifp == ifp &&
(kgdb) list
326              * first one on the interface, if possible.
327              */
328             dst = ((struct sockaddr_in *)&ifr->ifr_addr)->sin_addr;
329             IN_IFADDR_RLOCK();
330             LIST_FOREACH(iap, INADDR_HASH(dst.s_addr), ia_hash) {
331                     if (iap->ia_ifp == ifp &&
332                         iap->ia_addr.sin_addr.s_addr == dst.s_addr) {
333                             if (td == NULL || prison_check_ip4(td->td_ucred,
334                                 &dst) == 0)
335                                     ia = iap;
(kgdb) p iap
$1 = (struct in_ifaddr *) 0xdeadc0de

Note, some types of the crashes were possible to see only after applying the patches that fixed another types of the crashes -- the crashes have different probabylity of occurance.
>How-To-Repeat:
Run several copies of the script like below simultaneously.

while true; do
        ifconfig $IFACE  alias $IP
        ifconfig $IFACE -alias $IP
done

>Fix:
See the attaches patches that fix problems 1 - 4. There is no patch for the crash 5.

Also I am not sure about the patch for the 2nd crash. The patch just replaces LLTABLE_RLOCK with LLTABLE_WLOCK, but there might be some reasons to use RLOCK there?

Patch attached with submission follows:

# This is a shell archive.  Save it in a file, remove anything before
# this line, and then unpack it by entering "sh file".  Note, it may
# create directories; files and directories will be owned by you and
# have default permissions.
#
# This archive contains:
#
#	patches
#	patches/4-in.c.in_ifinit.patch
#	patches/1-in.c.in_control.patch
#	patches/3-rtsock.c.sysctl_iflist.patch
#	patches/2-if_llatbl.c.lltable_prefix_free.patch
#
echo c - patches
mkdir -p patches > /dev/null 2>&1
echo x - patches/4-in.c.in_ifinit.patch
sed 's/^X//' >patches/4-in.c.in_ifinit.patch << 'b87c9037ae1b4c044e29385c8651c6ad'
X--- sys/netinet/in.c.in_control	2010-04-18 21:00:37.000000000 +0300
X+++ sys/netinet/in.c	2010-04-20 13:08:41.000000000 +0300
X@@ -836,19 +836,25 @@ in_ifinit(struct ifnet *ifp, struct in_i
X     int scrub)
X {
X 	register u_long i = ntohl(sin->sin_addr.s_addr);
X+	register struct in_ifaddr *iap;
X 	struct sockaddr_in oldaddr;
X 	int s = splimp(), flags = RTF_UP, error = 0;
X 
X 	oldaddr = ia->ia_addr;
X-	if (oldaddr.sin_family == AF_INET)
X-		LIST_REMOVE(ia, ia_hash);
X+	IN_IFADDR_WLOCK();
X+	if (oldaddr.sin_family == AF_INET) {
X+		LIST_FOREACH(iap, INADDR_HASH(oldaddr.sin_addr.s_addr), ia_hash) {
X+			if (iap == ia) {
X+				LIST_REMOVE(ia, ia_hash);
X+				break;
X+			}
X+		}
X+	}
X 	ia->ia_addr = *sin;
X-	if (ia->ia_addr.sin_family == AF_INET) {
X-		IN_IFADDR_WLOCK();
X+	if (ia->ia_addr.sin_family == AF_INET) 
X 		LIST_INSERT_HEAD(INADDR_HASH(ia->ia_addr.sin_addr.s_addr),
X 		    ia, ia_hash);
X-		IN_IFADDR_WUNLOCK();
X-	}
X+	IN_IFADDR_WUNLOCK();
X 	/*
X 	 * Give the interface a chance to initialize
X 	 * if this is its first address,
b87c9037ae1b4c044e29385c8651c6ad
echo x - patches/1-in.c.in_control.patch
sed 's/^X//' >patches/1-in.c.in_control.patch << 'f1f1dfd62b4c5b1e308fc943ca5e7e21'
X--- sys/netinet/in.c.orig	2010-04-16 15:15:07.000000000 +0300
X+++ sys/netinet/in.c	2010-04-18 17:22:57.000000000 +0300
X@@ -601,8 +601,17 @@ in_control(struct socket *so, u_long cmd
X 	}
X 
X 	IF_ADDR_LOCK(ifp);
X-	TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link);
X+	TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
X+		if (&ia->ia_ifa == ifa) {
X+			TAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifa_link);
X+			break;
X+		}
X+	}
X 	IF_ADDR_UNLOCK(ifp);
X+	if (ifa == NULL) {
X+		error = EADDRNOTAVAIL;
X+		goto out;
X+	}
X 	ifa_free(&ia->ia_ifa);				/* if_addrhead */
X 
X 	IN_IFADDR_WLOCK();
f1f1dfd62b4c5b1e308fc943ca5e7e21
echo x - patches/3-rtsock.c.sysctl_iflist.patch
sed 's/^X//' >patches/3-rtsock.c.sysctl_iflist.patch << 'a11a2404faeb3a60e843b7c43fe14382'
X--- sys/net/rtsock.c.orig	2010-04-19 08:19:48.000000000 +0300
X+++ sys/net/rtsock.c	2010-04-19 08:26:02.000000000 +0300
X@@ -1380,6 +1380,7 @@ sysctl_iflist(int af, struct walkarg *w)
X 	TAILQ_FOREACH(ifp, &V_ifnet, if_link) {
X 		if (w->w_arg && w->w_arg != ifp->if_index)
X 			continue;
X+		IF_ADDR_LOCK(ifp);
X 		ifa = ifp->if_addr;
X 		info.rti_info[RTAX_IFP] = ifa->ifa_addr;
X 		len = rt_msg2(RTM_IFINFO, &info, NULL, w);
X@@ -1419,10 +1420,13 @@ sysctl_iflist(int af, struct walkarg *w)
X 					goto done;
X 			}
X 		}
X+		IF_ADDR_UNLOCK(ifp);
X 		info.rti_info[RTAX_IFA] = info.rti_info[RTAX_NETMASK] =
X 			info.rti_info[RTAX_BRD] = NULL;
X 	}
X done:
X+	if (ifp)
X+		IF_ADDR_UNLOCK(ifp);
X 	IFNET_RUNLOCK();
X 	return (error);
X }
a11a2404faeb3a60e843b7c43fe14382
echo x - patches/2-if_llatbl.c.lltable_prefix_free.patch
sed 's/^X//' >patches/2-if_llatbl.c.lltable_prefix_free.patch << '67f2a3bc2474cc4d802304e3a8c00915'
X--- sys/net/if_llatbl.c.orig	2010-04-18 22:38:58.000000000 +0300
X+++ sys/net/if_llatbl.c	2010-04-18 22:39:13.000000000 +0300
X@@ -209,14 +209,14 @@ lltable_prefix_free(int af, struct socka
X {
X 	struct lltable *llt;
X 
X-	LLTABLE_RLOCK();
X+	LLTABLE_WLOCK();
X 	SLIST_FOREACH(llt, &V_lltables, llt_link) {
X 		if (llt->llt_af != af)
X 			continue;
X 
X 		llt->llt_prefix_free(llt, prefix, mask);
X 	}
X-	LLTABLE_RUNLOCK();
X+	LLTABLE_WUNLOCK();
X }
X 
X 
67f2a3bc2474cc4d802304e3a8c00915
exit



>Release-Note:
>Audit-Trail:
>Unformatted:



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201005021727.o42HRr40025514>