From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 01:27:58 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id E8D0F1065672; Sun, 1 Mar 2009 01:27:58 +0000 (UTC) (envelope-from thompsa@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id CBF5B8FC24; Sun, 1 Mar 2009 01:27:58 +0000 (UTC) (envelope-from thompsa@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n211Rw7L026643; Sun, 1 Mar 2009 01:27:58 GMT (envelope-from thompsa@svn.freebsd.org) Received: (from thompsa@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n211Rwke026639; Sun, 1 Mar 2009 01:27:58 GMT (envelope-from thompsa@svn.freebsd.org) Message-Id: <200903010127.n211Rwke026639@svn.freebsd.org> From: Andrew Thompson Date: Sun, 1 Mar 2009 01:27:58 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189197 - user/thompsa/vaptq/sys/net80211 X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 01:27:59 -0000 Author: thompsa Date: Sun Mar 1 01:27:58 2009 New Revision: 189197 URL: http://svn.freebsd.org/changeset/base/189197 Log: Add taskqueue events for promisc and mcast update. Modified: user/thompsa/vaptq/sys/net80211/ieee80211.c user/thompsa/vaptq/sys/net80211/ieee80211_proto.c user/thompsa/vaptq/sys/net80211/ieee80211_scan_sta.c user/thompsa/vaptq/sys/net80211/ieee80211_var.h Modified: user/thompsa/vaptq/sys/net80211/ieee80211.c ============================================================================== --- user/thompsa/vaptq/sys/net80211/ieee80211.c Sat Feb 28 22:58:52 2009 (r189196) +++ user/thompsa/vaptq/sys/net80211/ieee80211.c Sun Mar 1 01:27:58 2009 (r189197) @@ -624,9 +624,11 @@ ieee80211_syncifflag_locked(struct ieee8 /* XXX should we return 1/0 and let caller do this? */ if (ifp->if_drv_flags & IFF_DRV_RUNNING) { if (flag == IFF_PROMISC) - ic->ic_update_promisc(ifp); + taskqueue_enqueue(ic->ic_tq, + &ic->ic_promisc_task); else if (flag == IFF_ALLMULTI) - ic->ic_update_mcast(ifp); + taskqueue_enqueue(ic->ic_tq, + &ic->ic_mcast_task); } } } Modified: user/thompsa/vaptq/sys/net80211/ieee80211_proto.c ============================================================================== --- user/thompsa/vaptq/sys/net80211/ieee80211_proto.c Sat Feb 28 22:58:52 2009 (r189196) +++ user/thompsa/vaptq/sys/net80211/ieee80211_proto.c Sun Mar 1 01:27:58 2009 (r189197) @@ -97,6 +97,8 @@ const char *ieee80211_wme_acnames[] = { }; static void parent_updown(void *, int); +static void update_mcast(void *, int); +static void update_promisc(void *, int); static void ieee80211_newstate_cb(void *, int); static int ieee80211_newstate_cb_locked(struct ieee80211vap *, enum ieee80211_state, int); @@ -134,6 +136,8 @@ ieee80211_proto_attach(struct ieee80211c ic->ic_protmode = IEEE80211_PROT_CTSONLY; TASK_INIT(&ic->ic_parent_task, 0, parent_updown, ifp); + TASK_INIT(&ic->ic_mcast_task, 0, update_mcast, ic); + TASK_INIT(&ic->ic_promisc_task, 0, update_promisc, ic); ic->ic_wme.wme_hipri_switch_hysteresis = AGGRESSIVE_MODE_SWITCH_HYSTERESIS; @@ -1076,6 +1080,24 @@ parent_updown(void *arg, int npending) parent->if_ioctl(parent, SIOCSIFFLAGS, NULL); } +static void +update_mcast(void *arg, int npending) +{ + struct ieee80211com *ic = arg; + struct ifnet *parent = ic->ic_ifp; + + ic->ic_update_mcast(parent); +} + +static void +update_promisc(void *arg, int npending) +{ + struct ieee80211com *ic = arg; + struct ifnet *parent = ic->ic_ifp; + + ic->ic_update_promisc(parent); +} + /* * Block until the parent is in a known state. This is * used after any operations that dispatch a task (e.g. @@ -1084,7 +1106,9 @@ parent_updown(void *arg, int npending) void ieee80211_waitfor_parent(struct ieee80211com *ic) { - taskqueue_drain(taskqueue_thread, &ic->ic_parent_task); + taskqueue_drain(ic->ic_tq, &ic->ic_parent_task); + taskqueue_drain(ic->ic_tq, &ic->ic_mcast_task); + taskqueue_drain(ic->ic_tq, &ic->ic_promisc_task); } /* @@ -1651,8 +1675,7 @@ ieee80211_new_state_locked(struct ieee80 struct ieee80211com *ic = vap->iv_ic; struct ieee80211vap *vp; enum ieee80211_state ostate; - int nrunning, nscanning, rc; - int forcesync = 0; + int forcesync, nrunning, nscanning, rc; IEEE80211_LOCK_ASSERT(ic); @@ -1669,6 +1692,7 @@ ieee80211_new_state_locked(struct ieee80 } } ostate = vap->iv_state; + forcesync = 0; IEEE80211_DPRINTF(vap, IEEE80211_MSG_STATE, "%s: %s -> %s (nrunning %d nscanning %d)\n", __func__, ieee80211_state_name[ostate], ieee80211_state_name[nstate], Modified: user/thompsa/vaptq/sys/net80211/ieee80211_scan_sta.c ============================================================================== --- user/thompsa/vaptq/sys/net80211/ieee80211_scan_sta.c Sat Feb 28 22:58:52 2009 (r189196) +++ user/thompsa/vaptq/sys/net80211/ieee80211_scan_sta.c Sun Mar 1 01:27:58 2009 (r189197) @@ -32,10 +32,11 @@ __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include -#include +#include #include #include - +#include + #include #include @@ -1609,7 +1610,7 @@ ap_force_promisc(struct ieee80211com *ic IEEE80211_LOCK(ic); /* set interface into promiscuous mode */ ifp->if_flags |= IFF_PROMISC; - ic->ic_update_promisc(ifp); + taskqueue_enqueue(ic->ic_tq, &ic->ic_promisc_task); IEEE80211_UNLOCK(ic); } Modified: user/thompsa/vaptq/sys/net80211/ieee80211_var.h ============================================================================== --- user/thompsa/vaptq/sys/net80211/ieee80211_var.h Sat Feb 28 22:58:52 2009 (r189196) +++ user/thompsa/vaptq/sys/net80211/ieee80211_var.h Sun Mar 1 01:27:58 2009 (r189197) @@ -119,6 +119,8 @@ struct ieee80211com { struct callout ic_inact; /* inactivity processing */ struct taskqueue *ic_tq; /* deferred state thread */ struct task ic_parent_task; /* deferred parent processing */ + struct task ic_promisc_task;/* deferred promisc update */ + struct task ic_mcast_task; /* deferred mcast update */ uint32_t ic_flags; /* state flags */ uint32_t ic_flags_ext; /* extended state flags */ From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 03:07:07 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id CEAE4106566B; Sun, 1 Mar 2009 03:07:07 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id A2B218FC0C; Sun, 1 Mar 2009 03:07:07 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n21377Pi028747; Sun, 1 Mar 2009 03:07:07 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n21377W9028746; Sun, 1 Mar 2009 03:07:07 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010307.n21377W9028746@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 03:07:07 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189198 - user/kmacy/releng_7_fast_net X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 03:07:08 -0000 Author: kmacy Date: Sun Mar 1 03:07:07 2009 New Revision: 189198 URL: http://svn.freebsd.org/changeset/base/189198 Log: create branch for releng 7 backport of HEAD_fast_net changes Added: user/kmacy/releng_7_fast_net/ - copied from r189197, stable/7/ From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 03:32:28 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id CB35F106566C; Sun, 1 Mar 2009 03:32:28 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id B88728FC12; Sun, 1 Mar 2009 03:32:28 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n213WSNm029230; Sun, 1 Mar 2009 03:32:28 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n213WSha029225; Sun, 1 Mar 2009 03:32:28 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010332.n213WSha029225@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 03:32:28 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189199 - user/kmacy/releng_7_fast_net/sys/netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 03:32:29 -0000 Author: kmacy Date: Sun Mar 1 03:32:28 2009 New Revision: 189199 URL: http://svn.freebsd.org/changeset/base/189199 Log: IFC HEAD_fast_net 186573: cache rtentry in inpcb for connected sockets Modified: user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.c user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c user/kmacy/releng_7_fast_net/sys/netinet/tcp_usrreq.c Modified: user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.c Sun Mar 1 03:07:07 2009 (r189198) +++ user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.c Sun Mar 1 03:32:28 2009 (r189199) @@ -489,6 +489,39 @@ in_pcbbind_setup(struct inpcb *inp, stru return (0); } +void +in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro) +{ + struct sockaddr_in *sin; + + INP_WLOCK_ASSERT(inp); + bzero(sro, sizeof(*sro)); + sin = (struct sockaddr_in *)&sro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(struct sockaddr_in); + sin->sin_addr.s_addr = faddr; + /* + * If route is known our src addr is taken from the i/f, + * else punt. + * + * Find out route to destination. + */ + if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) { +#ifdef RADIX_MPATH + rtalloc_mpath_fib(sro, ntohl(faddr->s_addr), + inp->inp_inc.inc_fibnum); +#else + in_rtalloc_ign(sro, 0, inp->inp_inc.inc_fibnum); +#endif + } + + if (sro->ro_rt != NULL) { + inp->inp_rt = sro->ro_rt; + inp->inp_vflag |= INP_RT_VALID; + + } +} + /* * Connect from a socket to a specified address. * Both address and port must be specified in argument sin. @@ -501,6 +534,7 @@ in_pcbconnect(struct inpcb *inp, struct u_short lport, fport; in_addr_t laddr, faddr; int anonport, error; + struct route sro; INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo); INP_WLOCK_ASSERT(inp); @@ -524,6 +558,7 @@ in_pcbconnect(struct inpcb *inp, struct } } + in_pcbrtalloc(inp, faddr, &sro); /* Commit the remaining changes. */ inp->inp_lport = lport; inp->inp_laddr.s_addr = laddr; @@ -861,6 +896,12 @@ in_pcbdisconnect(struct inpcb *inp) INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo); INP_WLOCK_ASSERT(inp); + if (inp->inp_vflag & INP_RT_VALID) { + RTFREE(inp->inp_rt); + inp->inp_rt = NULL; + inp->inp_vflag &= ~INP_RT_VALID; + } + inp->inp_faddr.s_addr = INADDR_ANY; inp->inp_fport = 0; in_pcbrehash(inp); @@ -900,6 +941,12 @@ in_pcbfree(struct inpcb *inp) INP_INFO_WLOCK_ASSERT(ipi); INP_WLOCK_ASSERT(inp); + if (inp->inp_vflag & INP_RT_VALID) { + RTFREE(inp->inp_rt); + inp->inp_rt = NULL; + inp->inp_vflag &= ~INP_RT_VALID; + } + #ifdef IPSEC ipsec4_delete_pcbpolicy(inp); #endif /*IPSEC*/ Modified: user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Sun Mar 1 03:07:07 2009 (r189198) +++ user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Sun Mar 1 03:32:28 2009 (r189199) @@ -160,9 +160,9 @@ struct inpcb { u_char inp_ip_ttl; /* (i) time to live proto */ u_char inp_ip_p; /* (c) protocol proto */ u_char inp_ip_minttl; /* (i) minimum TTL or drop */ - uint32_t inp_ispare1; /* (x) connection id / queue id */ - void *inp_pspare; /* (x) rtentry / general use */ - struct ucred *inp_cred; /* (c) cache of socket cred */ + u_int inp_refcount; /* (i) refcount */ + void *inp_pspare[1]; /* L2 information */ + struct rtentry *inp_rt; /* L3 information */ /* Local and foreign ports, local and foreign addr. */ struct in_conninfo inp_inc; /* (i/p) list for PCB's local port */ @@ -317,6 +317,9 @@ struct inpcbinfo { #define INP_RLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_RLOCKED) #define INP_WLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_WLOCKED) #define INP_UNLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_UNLOCKED) +#define INP_TRY_UPGRADE(inp) rw_try_upgrade(&(inp)->inp_lock) +#define INP_DOWNGRADE(inp) rw_downgrade(&(inp)->inp_lock) +#define INP_WLOCKED(inp) rw_wowned(&(inp)->inp_lock) #ifdef _KERNEL /* @@ -414,6 +417,7 @@ void inp_4tuple_get(struct inpcb *inp, #define INP_ONESBCAST 0x10 /* send all-ones broadcast */ #define INP_DROPPED 0x20 /* protocol drop flag */ #define INP_SOCKREF 0x40 /* strong socket reference */ +#define INP_RT_VALID 0x80 /* rtentry is set */ /* * Flags for inp_flag. @@ -499,6 +503,7 @@ void in_pcbdisconnect(struct inpcb *); void in_pcbdrop(struct inpcb *); void in_pcbfree(struct inpcb *); int in_pcbinshash(struct inpcb *); +void in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro); struct inpcb * in_pcblookup_local(struct inpcbinfo *, struct in_addr, u_short, int, struct ucred *); Modified: user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c Sun Mar 1 03:07:07 2009 (r189198) +++ user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c Sun Mar 1 03:32:28 2009 (r189199) @@ -110,11 +110,13 @@ ip_output(struct mbuf *m, struct mbuf *o int hlen = sizeof (struct ip); int mtu; int len, error = 0; + int neednewroute = 0; struct sockaddr_in *dst = NULL; /* keep compiler happy */ struct in_ifaddr *ia = NULL; int isbroadcast, sw_csum; struct route iproute; struct in_addr odst; + struct sockaddr_in *sin; #ifdef IPFIREWALL_FORWARD struct m_tag *fwd_tag = NULL; #endif @@ -128,6 +130,16 @@ ip_output(struct mbuf *m, struct mbuf *o if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); INP_LOCK_ASSERT(inp); + if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) { + if (inp->inp_rt->rt_flags & RTF_UP) { + sin = (struct sockaddr_in *)&ro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(struct sockaddr_in); + sin->sin_addr.s_addr = inp->inp_faddr.s_addr; + ro->ro_rt = inp->inp_rt; + } else + neednewroute = 1; + } } if (opt) { @@ -170,7 +182,8 @@ again: if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 || dst->sin_family != AF_INET || dst->sin_addr.s_addr != ip->ip_dst.s_addr)) { - RTFREE(ro->ro_rt); + if (inp == NULL || (ro->ro_rt != inp->inp_rt)) + RTFREE(ro->ro_rt); ro->ro_rt = (struct rtentry *)NULL; } #ifdef IPFIREWALL_FORWARD @@ -595,8 +608,20 @@ passout: ipstat.ips_fragmented++; done: - if (ro == &iproute && ro->ro_rt) { - RTFREE(ro->ro_rt); + if (ro == &iproute && ro->ro_rt != NULL) { + int wlocked; + + if (inp == NULL || (inp->inp_vflag & INP_RT_VALID) == 0) + RTFREE(ro->ro_rt); + else if (neednewroute && ro->ro_rt != inp->inp_rt) { + wlocked = INP_WLOCKED(inp); + if (!wlocked && INP_TRY_UPGRADE(inp) == 0) + return (error); + RTFREE(inp->inp_rt); + inp->inp_rt = ro->ro_rt; + if (!wlocked) + INP_DOWNGRADE(inp); + } } return (error); bad: Modified: user/kmacy/releng_7_fast_net/sys/netinet/tcp_usrreq.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/tcp_usrreq.c Sun Mar 1 03:07:07 2009 (r189198) +++ user/kmacy/releng_7_fast_net/sys/netinet/tcp_usrreq.c Sun Mar 1 03:32:28 2009 (r189199) @@ -1087,6 +1087,7 @@ tcp_connect(struct tcpcb *tp, struct soc struct inpcb *inp = tp->t_inpcb, *oinp; struct socket *so = inp->inp_socket; struct in_addr laddr; + struct route sro; u_short lport; int error; @@ -1115,6 +1116,7 @@ tcp_connect(struct tcpcb *tp, struct soc inp->inp_laddr = laddr; in_pcbrehash(inp); + in_pcbrtalloc(inp, inp->inp_faddr.s_addr, &sro); /* * Compute window scaling to request: * Scale to fit into sweet spot. See tcp_syncache.c. From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 04:39:57 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 519A1106564A; Sun, 1 Mar 2009 04:39:57 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 3D61A8FC19; Sun, 1 Mar 2009 04:39:57 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n214dvYv030408; Sun, 1 Mar 2009 04:39:57 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n214duxo030398; Sun, 1 Mar 2009 04:39:56 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010439.n214duxo030398@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 04:39:56 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189200 - in user/kmacy/releng_7_fast_net/sys: amd64/include arm/include dev/bce dev/mxge i386/include ia64/include powerpc/include sparc64/include sun4v/include X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 04:39:57 -0000 Author: kmacy Date: Sun Mar 1 04:39:56 2009 New Revision: 189200 URL: http://svn.freebsd.org/changeset/base/189200 Log: multiq step 1: - atomic_barriers: add atomic barriers to atomic.h on all architectures Modified: user/kmacy/releng_7_fast_net/sys/amd64/include/atomic.h user/kmacy/releng_7_fast_net/sys/arm/include/atomic.h user/kmacy/releng_7_fast_net/sys/dev/bce/if_bcereg.h user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge.c user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge_var.h user/kmacy/releng_7_fast_net/sys/i386/include/atomic.h user/kmacy/releng_7_fast_net/sys/ia64/include/atomic.h user/kmacy/releng_7_fast_net/sys/powerpc/include/atomic.h user/kmacy/releng_7_fast_net/sys/sparc64/include/atomic.h user/kmacy/releng_7_fast_net/sys/sun4v/include/atomic.h Modified: user/kmacy/releng_7_fast_net/sys/amd64/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/amd64/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/amd64/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -32,6 +32,10 @@ #error this file needs sys/cdefs.h as a prerequisite #endif +#define mb() __asm__ __volatile__ ("mfence;": : :"memory") +#define wmb() __asm__ __volatile__ ("sfence;": : :"memory") +#define rmb() __asm__ __volatile__ ("lfence;": : :"memory") + /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. Modified: user/kmacy/releng_7_fast_net/sys/arm/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/arm/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/arm/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -45,6 +45,10 @@ #include +#define mb() +#define wmb() +#define rmb() + #ifndef I32_bit #define I32_bit (1 << 7) /* IRQ disable */ #endif Modified: user/kmacy/releng_7_fast_net/sys/dev/bce/if_bcereg.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/dev/bce/if_bcereg.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/dev/bce/if_bcereg.h Sun Mar 1 04:39:56 2009 (r189200) @@ -557,6 +557,8 @@ #endif /* BCE_DEBUG */ + +#if __FreeBSD_version < 701101 #if defined(__i386__) || defined(__amd64__) #define mb() __asm volatile("mfence" ::: "memory") #define wmb() __asm volatile("sfence" ::: "memory") @@ -566,6 +568,7 @@ #define rmb() #define wmb() #endif +#endif /****************************************************************************/ /* Device identification definitions. */ Modified: user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge.c Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge.c Sun Mar 1 04:39:56 2009 (r189200) @@ -747,9 +747,9 @@ mxge_load_firmware_helper(mxge_softc_t * mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, inflate_buffer + i, min(256U, (unsigned)(fw_len - i))); - mb(); + wmb(); dummy = *sc->sram; - mb(); + wmb(); } *limit = fw_len; @@ -782,7 +782,7 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en /* clear confirmation addr */ confirm = (volatile uint32_t *)sc->cmd; *confirm = 0; - mb(); + wmb(); /* send an rdma command to the PCIe engine, and wait for the response in the confirmation address. The firmware should @@ -804,9 +804,9 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); mxge_pio_copy(submit, buf, 64); - mb(); + wmb(); DELAY(1000); - mb(); + wmb(); i = 0; while (*confirm != 0xffffffff && i < 20) { DELAY(1000); @@ -844,7 +844,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t buf->response_addr.high = htobe32(dma_high); mtx_lock(&sc->cmd_mtx); response->result = 0xffffffff; - mb(); + wmb(); mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); /* wait up to 20ms */ @@ -852,7 +852,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t for (sleep_total = 0; sleep_total < 20; sleep_total++) { bus_dmamap_sync(sc->cmd_dma.dmat, sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); - mb(); + wmb(); switch (be32toh(response->result)) { case 0: data->data0 = be32toh(response->data); @@ -980,7 +980,7 @@ mxge_load_firmware(mxge_softc_t *sc, int /* clear confirmation addr */ confirm = (volatile uint32_t *)sc->cmd; *confirm = 0; - mb(); + wmb(); /* send a reload command to the bootstrap MCP, and wait for the response in the confirmation address. The firmware should write a -1 there to indicate it is alive and well @@ -1005,9 +1005,9 @@ mxge_load_firmware(mxge_softc_t *sc, int submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); mxge_pio_copy(submit, buf, 64); - mb(); + wmb(); DELAY(1000); - mb(); + wmb(); i = 0; while (*confirm != 0xffffffff && i < 20) { DELAY(1000*10); @@ -1668,7 +1668,7 @@ mxge_submit_req_backwards(mxge_tx_ring_t idx = (starting_slot + cnt) & tx->mask; mxge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); - mb(); + wmb(); } } @@ -1694,14 +1694,14 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ last_flags = src->flags; src->flags = 0; - mb(); + wmb(); dst = dstp = &tx->lanai[idx]; srcp = src; if ((idx + cnt) < tx->mask) { for (i = 0; i < (cnt - 1); i += 2) { mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); - mb(); /* force write every 32 bytes */ + wmb(); /* force write every 32 bytes */ srcp += 2; dstp += 2; } @@ -1714,7 +1714,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ if (i < cnt) { /* submit the first request */ mxge_pio_copy(dstp, srcp, sizeof(*src)); - mb(); /* barrier before setting valid flag */ + wmb(); /* barrier before setting valid flag */ } /* re-write the last 32-bits with the valid flags */ @@ -1725,7 +1725,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ dst_ints+=3; *dst_ints = *src_ints; tx->req += cnt; - mb(); + wmb(); } #if IFCAP_TSO4 @@ -2144,12 +2144,12 @@ mxge_submit_8rx(volatile mcp_kreq_ether_ low = src->addr_low; src->addr_low = 0xffffffff; mxge_pio_copy(dst, src, 4 * sizeof (*src)); - mb(); + wmb(); mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); - mb(); + wmb(); src->addr_low = low; dst->addr_low = low; - mb(); + wmb(); } static int @@ -2736,7 +2736,7 @@ mxge_intr(void *arg) send_done_count = be32toh(stats->send_done_count); } if (sc->legacy_irq && mxge_deassert_wait) - mb(); + wmb(); } while (*((volatile uint8_t *) &stats->valid)); if (__predict_false(stats->stats_updated)) { @@ -3385,7 +3385,7 @@ mxge_close(mxge_softc_t *sc) callout_stop(&sc->co_hdl); sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; old_down_cnt = sc->down_cnt; - mb(); + wmb(); err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); if (err) { device_printf(sc->dev, "Couldn't bring down link\n"); @@ -3394,7 +3394,7 @@ mxge_close(mxge_softc_t *sc) /* wait for down irq */ DELAY(10 * sc->intr_coal_delay); } - mb(); + wmb(); if (old_down_cnt == sc->down_cnt) { device_printf(sc->dev, "never got down irq\n"); } Modified: user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge_var.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge_var.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/dev/mxge/if_mxge_var.h Sun Mar 1 04:39:56 2009 (r189200) @@ -282,6 +282,8 @@ struct mxge_media_type /* implement our own memory barriers, since bus_space_barrier cannot handle write-combining regions */ +#if __FreeBSD_version < 701101 + #if defined (__GNUC__) #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__ || #cpu(x86_64) || defined __x86_64__ #define mb() __asm__ __volatile__ ("sfence;": : :"memory") @@ -296,6 +298,8 @@ struct mxge_media_type #error "unknown compiler" #endif +#endif + static inline void mxge_pio_copy(volatile void *to_v, void *from_v, size_t size) { Modified: user/kmacy/releng_7_fast_net/sys/i386/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/i386/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/i386/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -32,6 +32,47 @@ #error this file needs sys/cdefs.h as a prerequisite #endif +#if defined(_KERNEL) +#include +#include +#define mb() __asm __volatile( \ + "testl %0,cpu_feature \n\ + je 2f \n\ + mfence \n\ +1: \n\ + .section .text.offpath \n\ +2: lock \n\ + addl $0,cpu_feature \n\ + jmp 1b \n\ + .text \n\ +" \ + : : "i"(CPUID_SSE2) : "memory") +#define wmb() __asm __volatile( \ + "testl %0,cpu_feature \n\ + je 2f \n\ + sfence \n\ +1: \n\ + .section .text.offpath \n\ +2: lock \n\ + addl $0,cpu_feature \n\ + jmp 1b \n\ + .text \n\ +" \ + : : "i"(CPUID_XMM) : "memory") +#define rmb() __asm __volatile( \ + "testl %0,cpu_feature \n\ + je 2f \n\ + lfence \n\ +1: \n\ + .section .text.offpath \n\ +2: lock \n\ + addl $0,cpu_feature \n\ + jmp 1b \n\ + .text \n\ +" \ + : : "i"(CPUID_SSE2) : "memory") +#endif + /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. Modified: user/kmacy/releng_7_fast_net/sys/ia64/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/ia64/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/ia64/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -29,6 +29,10 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ +#define mb() +#define wmb() +#define rmb() + /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and SMP safe. Modified: user/kmacy/releng_7_fast_net/sys/powerpc/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/powerpc/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/powerpc/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -39,6 +39,10 @@ #define __ATOMIC_BARRIER \ __asm __volatile("sync" : : : "memory") +#define mb() __ATOMIC_BARRIER +#define wmb() mb() +#define rmb() mb() + /* * atomic_add(p, v) * { *p += v; } Modified: user/kmacy/releng_7_fast_net/sys/sparc64/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/sparc64/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/sparc64/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -40,6 +40,10 @@ #define __ASI_ATOMIC ASI_P #endif +#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") +#define wmb() mb() +#define rmb() mb() + /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and multiple processors. See atomic(9) for details. Modified: user/kmacy/releng_7_fast_net/sys/sun4v/include/atomic.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/sun4v/include/atomic.h Sun Mar 1 03:32:28 2009 (r189199) +++ user/kmacy/releng_7_fast_net/sys/sun4v/include/atomic.h Sun Mar 1 04:39:56 2009 (r189200) @@ -33,6 +33,10 @@ #include +#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") +#define wmb() mb() +#define rmb() mb() + /* Userland needs different ASI's. */ #ifdef _KERNEL #define __ASI_ATOMIC ASI_N From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 04:43:30 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id B23C9106564A; Sun, 1 Mar 2009 04:43:30 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 9F0318FC08; Sun, 1 Mar 2009 04:43:30 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n214hU9K030507; Sun, 1 Mar 2009 04:43:30 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n214hUt5030504; Sun, 1 Mar 2009 04:43:30 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010443.n214hUt5030504@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 04:43:30 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189201 - in user/kmacy/releng_7_fast_net/sys: conf kern sys X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 04:43:31 -0000 Author: kmacy Date: Sun Mar 1 04:43:30 2009 New Revision: 189201 URL: http://svn.freebsd.org/changeset/base/189201 Log: multiq step 2: - buf_ring: add 'buf_ring', a generic lockless ring-buffer Added: user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c (contents, props changed) user/kmacy/releng_7_fast_net/sys/sys/buf_ring.h (contents, props changed) Modified: user/kmacy/releng_7_fast_net/sys/conf/files Modified: user/kmacy/releng_7_fast_net/sys/conf/files ============================================================================== --- user/kmacy/releng_7_fast_net/sys/conf/files Sun Mar 1 04:39:56 2009 (r189200) +++ user/kmacy/releng_7_fast_net/sys/conf/files Sun Mar 1 04:43:30 2009 (r189201) @@ -1489,6 +1489,7 @@ kern/subr_acl_posix1e.c standard kern/subr_autoconf.c standard kern/subr_blist.c standard kern/subr_bus.c standard +kern/subr_bufring.c standard kern/subr_clock.c standard kern/subr_devstat.c standard kern/subr_disk.c standard Added: user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c Sun Mar 1 04:43:30 2009 (r189201) @@ -0,0 +1,136 @@ +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * + ***************************************************************************/ +#include +__FBSDID("$FreeBSD$"); + + +#include +#include +#include +#include +#include +#include + + +struct buf_ring * +buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock) +{ + struct buf_ring *br; + + KASSERT(powerof2(count), ("buf ring must be size power of 2")); + + br = malloc(sizeof(struct buf_ring) + count*sizeof(caddr_t), + type, flags|M_ZERO); + if (br == NULL) + return (NULL); +#ifdef DEBUG_BUFRING + br->br_lock = lock; +#endif + br->br_prod_size = br->br_cons_size = count; + br->br_prod_mask = br->br_cons_mask = count-1; + br->br_prod_head = br->br_cons_head = 0; + br->br_prod_tail = br->br_cons_tail = 0; + + return (br); +} + +void +buf_ring_free(struct buf_ring *br, struct malloc_type *type) +{ + free(br, type); +} +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * + ***************************************************************************/ +#include +__FBSDID("$FreeBSD$"); + + +#include +#include +#include +#include +#include +#include + + +struct buf_ring * +buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock) +{ + struct buf_ring *br; + + KASSERT(powerof2(count), ("buf ring must be size power of 2")); + + br = malloc(sizeof(struct buf_ring) + count*sizeof(caddr_t), + type, flags|M_ZERO); + if (br == NULL) + return (NULL); +#ifdef DEBUG_BUFRING + br->br_lock = lock; +#endif + br->br_prod_size = br->br_cons_size = count; + br->br_prod_mask = br->br_cons_mask = count-1; + br->br_prod_head = br->br_cons_head = 0; + br->br_prod_tail = br->br_cons_tail = 0; + + return (br); +} + +void +buf_ring_free(struct buf_ring *br, struct malloc_type *type) +{ + free(br, type); +} Added: user/kmacy/releng_7_fast_net/sys/sys/buf_ring.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/releng_7_fast_net/sys/sys/buf_ring.h Sun Mar 1 04:43:30 2009 (r189201) @@ -0,0 +1,500 @@ +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + ***************************************************************************/ + +#ifndef _SYS_BUF_RING_H_ +#define _SYS_BUF_RING_H_ + +#include + +#if defined(INVARIANTS) && !defined(DEBUG_BUFRING) +#define DEBUG_BUFRING 1 +#endif + +#ifdef DEBUG_BUFRING +#include +#include +#endif + +struct buf_ring { + volatile uint32_t br_prod_head; + volatile uint32_t br_prod_tail; + int br_prod_size; + int br_prod_mask; + /* + * Pad out to next L2 cache line + */ + uint64_t _pad0[14]; + + volatile uint32_t br_cons_head; + volatile uint32_t br_cons_tail; + int br_cons_size; + int br_cons_mask; + + /* + * Pad out to next L2 cache line + */ + uint64_t _pad1[14]; +#ifdef DEBUG_BUFRING + struct mtx *br_lock; +#endif + void *br_ring[0]; +}; + + +static __inline int +buf_ring_enqueue(struct buf_ring *br, void *buf) +{ + uint32_t prod_head, prod_next; + uint32_t cons_tail; + int success; +#ifdef DEBUG_BUFRING + int i; + for (i = br->br_cons_head; i != br->br_prod_head; + i = ((i + 1) & br->br_cons_mask)) + if(br->br_ring[i] == buf) + panic("buf=%p already enqueue at %d prod=%d cons=%d", + buf, i, br->br_prod_tail, br->br_cons_tail); +#endif + critical_enter(); + do { + prod_head = br->br_prod_head; + cons_tail = br->br_cons_tail; + + prod_next = (prod_head + 1) & br->br_prod_mask; + + if (prod_next == cons_tail) { + critical_exit(); + return (ENOSPC); + } + + success = atomic_cmpset_int(&br->br_prod_head, prod_head, + prod_next); + } while (success == 0); +#ifdef DEBUG_BUFRING + if (br->br_ring[prod_head] != NULL) + panic("dangling value in enqueue"); +#endif + br->br_ring[prod_head] = buf; + wmb(); + + /* + * If there are other enqueues in progress + * that preceeded us, we need to wait for them + * to complete + */ + while (br->br_prod_tail != prod_head) + cpu_spinwait(); + br->br_prod_tail = prod_next; + mb(); + critical_exit(); + return (0); +} + +/* + * multi-consumer safe dequeue + * + */ +static __inline void * +buf_ring_dequeue_mc(struct buf_ring *br) +{ + uint32_t cons_head, cons_next; + uint32_t prod_tail; + void *buf; + int success; + + critical_enter(); + do { + cons_head = br->br_cons_head; + prod_tail = br->br_prod_tail; + + cons_next = (cons_head + 1) & br->br_cons_mask; + + if (cons_head == prod_tail) { + critical_exit(); + return (NULL); + } + + success = atomic_cmpset_int(&br->br_cons_head, cons_head, + cons_next); + } while (success == 0); + + buf = br->br_ring[cons_head]; +#ifdef DEBUG_BUFRING + br->br_ring[cons_head] = NULL; +#endif + mb(); + + /* + * If there are other dequeues in progress + * that preceeded us, we need to wait for them + * to complete + */ + while (br->br_cons_tail != cons_head) + cpu_spinwait(); + + br->br_cons_tail = cons_next; + mb(); + critical_exit(); + + return (buf); +} + +/* + * Single-Consumer dequeue for uses where dequeue + * is protected by a lock + */ +static __inline void * +buf_ring_dequeue_sc(struct buf_ring *br) +{ + uint32_t cons_head, cons_next; + uint32_t prod_tail; + void *buf; + + critical_enter(); + cons_head = br->br_cons_head; + prod_tail = br->br_prod_tail; + + cons_next = (cons_head + 1) & br->br_cons_mask; + + if (cons_head == prod_tail) { + critical_exit(); + return (NULL); + } + + br->br_cons_head = cons_next; + buf = br->br_ring[cons_head]; + mb(); + +#ifdef DEBUG_BUFRING + br->br_ring[cons_head] = NULL; + if (!mtx_owned(br->br_lock)) + panic("lock not held on single consumer dequeue"); + if (br->br_cons_tail != cons_head) + panic("inconsistent list cons_tail=%d cons_head=%d", + br->br_cons_tail, cons_head); +#endif + br->br_cons_tail = cons_next; + mb(); + critical_exit(); + return (buf); +} + +static __inline void * +buf_ring_peek(struct buf_ring *br) +{ + +#ifdef DEBUG_BUFRING + if ((br->br_lock != NULL) && !mtx_owned(br->br_lock)) + panic("lock not held on single consumer dequeue"); +#endif + mb(); + return (br->br_ring[br->br_cons_tail]); +} + +static __inline int +buf_ring_full(struct buf_ring *br) +{ + + return (((br->br_prod_head + 1) & br->br_prod_mask) == br->br_cons_tail); +} + +static __inline int +buf_ring_empty(struct buf_ring *br) +{ + + return (br->br_cons_head == br->br_prod_tail); +} + +static __inline int +buf_ring_count(struct buf_ring *br) +{ + + return ((br->br_prod_size + br->br_prod_tail - br->br_cons_tail) + & br->br_prod_mask); +} + +struct buf_ring *buf_ring_alloc(int count, struct malloc_type *type, int flags, + struct mtx *); +void buf_ring_free(struct buf_ring *br, struct malloc_type *type); + + + +#endif +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * $FreeBSD$ + * + ***************************************************************************/ + +#ifndef _SYS_BUF_RING_H_ +#define _SYS_BUF_RING_H_ + +#include + +#if defined(INVARIANTS) && !defined(DEBUG_BUFRING) +#define DEBUG_BUFRING 1 +#endif + +#ifdef DEBUG_BUFRING +#include +#include +#endif + +struct buf_ring { + volatile uint32_t br_prod_head; + volatile uint32_t br_prod_tail; + int br_prod_size; + int br_prod_mask; + /* + * Pad out to next L2 cache line + */ + uint64_t _pad0[14]; + + volatile uint32_t br_cons_head; + volatile uint32_t br_cons_tail; + int br_cons_size; + int br_cons_mask; + + /* + * Pad out to next L2 cache line + */ + uint64_t _pad1[14]; +#ifdef DEBUG_BUFRING + struct mtx *br_lock; +#endif + void *br_ring[0]; +}; + + +static __inline int +buf_ring_enqueue(struct buf_ring *br, void *buf) +{ + uint32_t prod_head, prod_next; + uint32_t cons_tail; + int success; +#ifdef DEBUG_BUFRING + int i; + for (i = br->br_cons_head; i != br->br_prod_head; + i = ((i + 1) & br->br_cons_mask)) + if(br->br_ring[i] == buf) + panic("buf=%p already enqueue at %d prod=%d cons=%d", + buf, i, br->br_prod_tail, br->br_cons_tail); +#endif + critical_enter(); + do { + prod_head = br->br_prod_head; + cons_tail = br->br_cons_tail; + + prod_next = (prod_head + 1) & br->br_prod_mask; + + if (prod_next == cons_tail) { + critical_exit(); + return (ENOSPC); + } + + success = atomic_cmpset_int(&br->br_prod_head, prod_head, + prod_next); + } while (success == 0); +#ifdef DEBUG_BUFRING + if (br->br_ring[prod_head] != NULL) + panic("dangling value in enqueue"); +#endif + br->br_ring[prod_head] = buf; + wmb(); + + /* + * If there are other enqueues in progress + * that preceeded us, we need to wait for them + * to complete + */ + while (br->br_prod_tail != prod_head) + cpu_spinwait(); + br->br_prod_tail = prod_next; + mb(); + critical_exit(); + return (0); +} + +/* + * multi-consumer safe dequeue + * + */ +static __inline void * +buf_ring_dequeue_mc(struct buf_ring *br) +{ + uint32_t cons_head, cons_next; + uint32_t prod_tail; + void *buf; + int success; + + critical_enter(); + do { + cons_head = br->br_cons_head; + prod_tail = br->br_prod_tail; + + cons_next = (cons_head + 1) & br->br_cons_mask; + + if (cons_head == prod_tail) { + critical_exit(); + return (NULL); + } + + success = atomic_cmpset_int(&br->br_cons_head, cons_head, + cons_next); + } while (success == 0); + + buf = br->br_ring[cons_head]; +#ifdef DEBUG_BUFRING + br->br_ring[cons_head] = NULL; +#endif + mb(); + + /* + * If there are other dequeues in progress + * that preceeded us, we need to wait for them + * to complete + */ + while (br->br_cons_tail != cons_head) + cpu_spinwait(); + + br->br_cons_tail = cons_next; + mb(); + critical_exit(); + + return (buf); +} + +/* + * Single-Consumer dequeue for uses where dequeue + * is protected by a lock + */ +static __inline void * +buf_ring_dequeue_sc(struct buf_ring *br) +{ + uint32_t cons_head, cons_next; + uint32_t prod_tail; + void *buf; + + critical_enter(); + cons_head = br->br_cons_head; + prod_tail = br->br_prod_tail; + + cons_next = (cons_head + 1) & br->br_cons_mask; + + if (cons_head == prod_tail) { + critical_exit(); + return (NULL); + } + + br->br_cons_head = cons_next; + buf = br->br_ring[cons_head]; + mb(); + +#ifdef DEBUG_BUFRING + br->br_ring[cons_head] = NULL; + if (!mtx_owned(br->br_lock)) + panic("lock not held on single consumer dequeue"); + if (br->br_cons_tail != cons_head) + panic("inconsistent list cons_tail=%d cons_head=%d", + br->br_cons_tail, cons_head); +#endif + br->br_cons_tail = cons_next; + mb(); + critical_exit(); + return (buf); +} + +static __inline void * +buf_ring_peek(struct buf_ring *br) +{ + +#ifdef DEBUG_BUFRING + if ((br->br_lock != NULL) && !mtx_owned(br->br_lock)) + panic("lock not held on single consumer dequeue"); +#endif + mb(); + return (br->br_ring[br->br_cons_tail]); +} + +static __inline int +buf_ring_full(struct buf_ring *br) +{ + + return (((br->br_prod_head + 1) & br->br_prod_mask) == br->br_cons_tail); +} + +static __inline int +buf_ring_empty(struct buf_ring *br) +{ + + return (br->br_cons_head == br->br_prod_tail); +} + +static __inline int +buf_ring_count(struct buf_ring *br) +{ + + return ((br->br_prod_size + br->br_prod_tail - br->br_cons_tail) + & br->br_prod_mask); +} + +struct buf_ring *buf_ring_alloc(int count, struct malloc_type *type, int flags, + struct mtx *); +void buf_ring_free(struct buf_ring *br, struct malloc_type *type); + + + +#endif From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 04:45:38 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 0509D1065673; Sun, 1 Mar 2009 04:45:38 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id E70018FC20; Sun, 1 Mar 2009 04:45:37 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n214jbNP030612; Sun, 1 Mar 2009 04:45:37 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n214jbb9030610; Sun, 1 Mar 2009 04:45:37 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010445.n214jbb9030610@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 04:45:37 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189202 - user/kmacy/releng_7_fast_net/sys/net X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 04:45:38 -0000 Author: kmacy Date: Sun Mar 1 04:45:37 2009 New Revision: 189202 URL: http://svn.freebsd.org/changeset/base/189202 Log: multiq step 3: - if_transmit: add if_transmit and if_qflush to struct ifnet to give high performance drivers the option of managing their own queues Modified: user/kmacy/releng_7_fast_net/sys/net/if.c user/kmacy/releng_7_fast_net/sys/net/if_var.h Modified: user/kmacy/releng_7_fast_net/sys/net/if.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if.c Sun Mar 1 04:43:30 2009 (r189201) +++ user/kmacy/releng_7_fast_net/sys/net/if.c Sun Mar 1 04:45:37 2009 (r189202) @@ -115,10 +115,11 @@ static void if_freemulti(struct ifmultia static void if_grow(void); static void if_init(void *); static void if_check(void *); -static void if_qflush(struct ifaltq *); +static void if_qflush(struct ifnet *); static void if_route(struct ifnet *, int flag, int fam); static int if_setflag(struct ifnet *, int, int, int *, int); static void if_slowtimo(void *); +static int if_transmit(struct ifnet *ifp, struct mbuf *m); static void if_unroute(struct ifnet *, int flag, int fam); static void link_rtrequest(int, struct rtentry *, struct rt_addrinfo *); static int if_rtdel(struct radix_node *, void *); @@ -128,6 +129,7 @@ static void if_start_deferred(void *cont static void do_link_state_change(void *, int); static int if_getgroup(struct ifgroupreq *, struct ifnet *); static int if_getgroupmembers(struct ifgroupreq *); + #ifdef INET6 /* * XXX: declare here to avoid to include many inet6 related files.. @@ -502,6 +504,28 @@ if_free_type(struct ifnet *ifp, u_char t free(ifp, M_IFNET); }; +void +ifq_attach(struct ifaltq *ifq, struct ifnet *ifp) +{ + + mtx_init(&ifq->ifq_mtx, ifp->if_xname, "if send queue", MTX_DEF); + + if (ifq->ifq_maxlen == 0) + ifq->ifq_maxlen = ifqmaxlen; + + ifq->altq_type = 0; + ifq->altq_disc = NULL; + ifq->altq_flags &= ALTQF_CANTCHANGE; + ifq->altq_tbr = NULL; + ifq->altq_ifp = ifp; +} + +void +ifq_detach(struct ifaltq *ifq) +{ + mtx_destroy(&ifq->ifq_mtx); +} + /* * Perform generic interface initalization tasks and attach the interface * to the list of "active" interfaces. @@ -542,7 +566,8 @@ if_attach(struct ifnet *ifp) getmicrotime(&ifp->if_lastchange); ifp->if_data.ifi_epoch = time_uptime; ifp->if_data.ifi_datalen = sizeof(struct if_data); - + ifp->if_transmit = if_transmit; + ifp->if_qflush = if_qflush; #ifdef MAC mac_init_ifnet(ifp); mac_create_ifnet(ifp); @@ -554,7 +579,7 @@ if_attach(struct ifnet *ifp) make_dev_alias(ifdev_byindex(ifp->if_index), "%s%d", net_cdevsw.d_name, ifp->if_index); - mtx_init(&ifp->if_snd.ifq_mtx, ifp->if_xname, "if send queue", MTX_DEF); + ifq_attach(&ifp->if_snd, ifp); /* * create a Link Level name for this device @@ -591,11 +616,6 @@ if_attach(struct ifnet *ifp) ifa->ifa_refcnt = 1; TAILQ_INSERT_HEAD(&ifp->if_addrhead, ifa, ifa_link); ifp->if_broadcastaddr = NULL; /* reliably crash if used uninitialized */ - ifp->if_snd.altq_type = 0; - ifp->if_snd.altq_disc = NULL; - ifp->if_snd.altq_flags &= ALTQF_CANTCHANGE; - ifp->if_snd.altq_tbr = NULL; - ifp->if_snd.altq_ifp = ifp; IFNET_WLOCK(); TAILQ_INSERT_TAIL(&ifnet, ifp, if_link); @@ -843,7 +863,7 @@ if_detach(struct ifnet *ifp) KNOTE_UNLOCKED(&ifp->if_klist, NOTE_EXIT); knlist_clear(&ifp->if_klist, 0); knlist_destroy(&ifp->if_klist); - mtx_destroy(&ifp->if_snd.ifq_mtx); + ifq_detach(&ifp->if_snd); IF_AFDATA_DESTROY(ifp); splx(s); } @@ -1387,7 +1407,8 @@ if_unroute(struct ifnet *ifp, int flag, TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) if (fam == PF_UNSPEC || (fam == ifa->ifa_addr->sa_family)) pfctlinput(PRC_IFDOWN, ifa->ifa_addr); - if_qflush(&ifp->if_snd); + ifp->if_qflush(ifp); + #ifdef DEV_CARP if (ifp->if_carp) carp_carpdev_state(ifp->if_carp); @@ -1515,10 +1536,12 @@ if_up(struct ifnet *ifp) * Flush an interface queue. */ static void -if_qflush(struct ifaltq *ifq) +if_qflush(struct ifnet *ifp) { struct mbuf *m, *n; - + struct ifaltq *ifq; + + ifq = &ifp->if_snd; IFQ_LOCK(ifq); #ifdef ALTQ if (ALTQ_IS_ENABLED(ifq)) @@ -2794,6 +2817,19 @@ if_start_deferred(void *context, int pen (ifp->if_start)(ifp); } +/* + * Backwards compatibility interface for drivers + * that have not implemented it + */ +static int +if_transmit(struct ifnet *ifp, struct mbuf *m) +{ + int error; + + IFQ_HANDOFF(ifp, m, error); + return (error); +} + int if_handoff(struct ifqueue *ifq, struct mbuf *m, struct ifnet *ifp, int adjust) { Modified: user/kmacy/releng_7_fast_net/sys/net/if_var.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_var.h Sun Mar 1 04:43:30 2009 (r189201) +++ user/kmacy/releng_7_fast_net/sys/net/if_var.h Sun Mar 1 04:45:37 2009 (r189202) @@ -186,7 +186,11 @@ struct ifnet { /* protected by if_addr_mtx */ void *if_pf_kif; void *if_lagg; /* lagg glue */ - void *if_pspare[10]; /* multiq/TOE 3; vimage 3; general use 4 */ + void *if_pspare[8]; /* multiq/TOE 3; vimage 3; general use 4 */ + void (*if_qflush) /* flush any queues */ + (struct ifnet *); + int (*if_transmit) /* initiate output routine */ + (struct ifnet *, struct mbuf *); int if_ispare[2]; /* general use 2 */ }; @@ -684,6 +688,9 @@ int ifioctl(struct socket *, u_long, cad int ifpromisc(struct ifnet *, int); struct ifnet *ifunit(const char *); +void ifq_attach(struct ifaltq *, struct ifnet *ifp); +void ifq_detach(struct ifaltq *); + struct ifaddr *ifa_ifwithaddr(struct sockaddr *); struct ifaddr *ifa_ifwithbroadaddr(struct sockaddr *); struct ifaddr *ifa_ifwithdstaddr(struct sockaddr *); From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 04:49:42 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id BD671106566B; Sun, 1 Mar 2009 04:49:42 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id A158A8FC0A; Sun, 1 Mar 2009 04:49:42 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n214ngLS030779; Sun, 1 Mar 2009 04:49:42 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n214ngmq030772; Sun, 1 Mar 2009 04:49:42 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010449.n214ngmq030772@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 04:49:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189203 - in user/kmacy/releng_7_fast_net/sys: net netgraph X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 04:49:43 -0000 Author: kmacy Date: Sun Mar 1 04:49:42 2009 New Revision: 189203 URL: http://svn.freebsd.org/changeset/base/189203 Log: multiq step 4: enable if_transmit: adds if_transmit and if_qflush to struct ifnet to give high performance drivers the option of managing their own queues Modified: user/kmacy/releng_7_fast_net/sys/net/if_ethersubr.c user/kmacy/releng_7_fast_net/sys/net/if_fddisubr.c user/kmacy/releng_7_fast_net/sys/net/if_fwsubr.c user/kmacy/releng_7_fast_net/sys/net/if_lagg.c user/kmacy/releng_7_fast_net/sys/net/if_tun.c user/kmacy/releng_7_fast_net/sys/net/if_vlan.c user/kmacy/releng_7_fast_net/sys/netgraph/ng_iface.c Modified: user/kmacy/releng_7_fast_net/sys/net/if_ethersubr.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_ethersubr.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/net/if_ethersubr.c Sun Mar 1 04:49:42 2009 (r189203) @@ -383,7 +383,6 @@ bad: if (m != NULL) int ether_output_frame(struct ifnet *ifp, struct mbuf *m) { - int error; #if defined(INET) || defined(INET6) struct ip_fw *rule = ip_dn_claim_rule(m); @@ -402,8 +401,7 @@ ether_output_frame(struct ifnet *ifp, st * Queue message on interface, update output statistics if * successful, and start output if interface not yet active. */ - IFQ_HANDOFF(ifp, m, error); - return (error); + return ((ifp->if_transmit)(ifp, m)); } #if defined(INET) || defined(INET6) Modified: user/kmacy/releng_7_fast_net/sys/net/if_fddisubr.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_fddisubr.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/net/if_fddisubr.c Sun Mar 1 04:49:42 2009 (r189203) @@ -336,7 +336,7 @@ fddi_output(ifp, m, dst, rt0) } } - IFQ_HANDOFF(ifp, m, error); + error = (ifp->if_transmit)(ifp, m); if (error) ifp->if_oerrors++; Modified: user/kmacy/releng_7_fast_net/sys/net/if_fwsubr.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_fwsubr.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/net/if_fwsubr.c Sun Mar 1 04:49:42 2009 (r189203) @@ -249,7 +249,7 @@ firewire_output(struct ifnet *ifp, struc */ enc->ul[0] = htonl(enc->ul[0]); - IFQ_HANDOFF(ifp, m, error); + error = (ifp->if_transmit)(ifp, m); return (error); } else { /* @@ -309,7 +309,7 @@ firewire_output(struct ifnet *ifp, struc enc->ul[0] = htonl(enc->ul[0]); enc->ul[1] = htonl(enc->ul[1]); - IFQ_HANDOFF(ifp, m, error); + error = (ifp->if_transmit)(ifp, m); if (error) { if (mtail) m_freem(mtail); Modified: user/kmacy/releng_7_fast_net/sys/net/if_lagg.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_lagg.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/net/if_lagg.c Sun Mar 1 04:49:42 2009 (r189203) @@ -1370,12 +1370,8 @@ out: int lagg_enqueue(struct ifnet *ifp, struct mbuf *m) { - int error = 0; - IFQ_HANDOFF(ifp, m, error); - if (error) - ifp->if_oerrors++; - return (error); + return (ifp->if_transmit)(ifp, m); } /* Modified: user/kmacy/releng_7_fast_net/sys/net/if_tun.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_tun.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/net/if_tun.c Sun Mar 1 04:49:42 2009 (r189203) @@ -648,7 +648,7 @@ tunoutput( } } - IFQ_HANDOFF(ifp, m0, error); + error = (ifp->if_transmit)(ifp, m0); if (error) { ifp->if_collisions++; return (ENOBUFS); Modified: user/kmacy/releng_7_fast_net/sys/net/if_vlan.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/net/if_vlan.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/net/if_vlan.c Sun Mar 1 04:49:42 2009 (r189203) @@ -864,7 +864,7 @@ vlan_start(struct ifnet *ifp) * Send it, precisely as ether_output() would have. * We are already running at splimp. */ - IFQ_HANDOFF(p, m, error); + error = (p->if_transmit)(p, m); if (!error) ifp->if_opackets++; else Modified: user/kmacy/releng_7_fast_net/sys/netgraph/ng_iface.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netgraph/ng_iface.c Sun Mar 1 04:45:37 2009 (r189202) +++ user/kmacy/releng_7_fast_net/sys/netgraph/ng_iface.c Sun Mar 1 04:49:42 2009 (r189203) @@ -400,7 +400,7 @@ ng_iface_output(struct ifnet *ifp, struc return (ENOBUFS); } *(sa_family_t *)m->m_data = dst->sa_family; - IFQ_HANDOFF(ifp, m, error); + error = (ifp->if_transmit)(ifp, m); } else error = ng_iface_send(ifp, m, dst->sa_family); From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 04:59:54 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id B2137106564A; Sun, 1 Mar 2009 04:59:54 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 9FC1D8FC0A; Sun, 1 Mar 2009 04:59:54 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n214xsq2031031; Sun, 1 Mar 2009 04:59:54 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n214xsnl031029; Sun, 1 Mar 2009 04:59:54 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010459.n214xsnl031029@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 04:59:54 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189205 - in user/kmacy/releng_7_fast_net/sys: kern netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 04:59:55 -0000 Author: kmacy Date: Sun Mar 1 04:59:54 2009 New Revision: 189205 URL: http://svn.freebsd.org/changeset/base/189205 Log: - fix accidental removal of inp_cred in 186573 - remove duplicate creation of subr_bufring.c in 189199 Modified: user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Modified: user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c Sun Mar 1 04:57:23 2009 (r189204) +++ user/kmacy/releng_7_fast_net/sys/kern/subr_bufring.c Sun Mar 1 04:59:54 2009 (r189205) @@ -66,71 +66,3 @@ buf_ring_free(struct buf_ring *br, struc { free(br, type); } -/************************************************************************** - * - * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. The name of Kip Macy nor the names of other - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * - * - ***************************************************************************/ -#include -__FBSDID("$FreeBSD$"); - - -#include -#include -#include -#include -#include -#include - - -struct buf_ring * -buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock) -{ - struct buf_ring *br; - - KASSERT(powerof2(count), ("buf ring must be size power of 2")); - - br = malloc(sizeof(struct buf_ring) + count*sizeof(caddr_t), - type, flags|M_ZERO); - if (br == NULL) - return (NULL); -#ifdef DEBUG_BUFRING - br->br_lock = lock; -#endif - br->br_prod_size = br->br_cons_size = count; - br->br_prod_mask = br->br_cons_mask = count-1; - br->br_prod_head = br->br_cons_head = 0; - br->br_prod_tail = br->br_cons_tail = 0; - - return (br); -} - -void -buf_ring_free(struct buf_ring *br, struct malloc_type *type) -{ - free(br, type); -} Modified: user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Sun Mar 1 04:57:23 2009 (r189204) +++ user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Sun Mar 1 04:59:54 2009 (r189205) @@ -161,6 +161,7 @@ struct inpcb { u_char inp_ip_p; /* (c) protocol proto */ u_char inp_ip_minttl; /* (i) minimum TTL or drop */ u_int inp_refcount; /* (i) refcount */ + struct ucred *inp_cred; /* (c) cache of socket cred */ void *inp_pspare[1]; /* L2 information */ struct rtentry *inp_rt; /* L3 information */ From owner-svn-src-user@FreeBSD.ORG Sun Mar 1 05:09:34 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id AF34B10675BB; Sun, 1 Mar 2009 05:09:34 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 9A8ED8FC12; Sun, 1 Mar 2009 05:09:34 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n2159Y9l031244; Sun, 1 Mar 2009 05:09:34 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n2159YAV031240; Sun, 1 Mar 2009 05:09:34 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903010509.n2159YAV031240@svn.freebsd.org> From: Kip Macy Date: Sun, 1 Mar 2009 05:09:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189206 - in user/kmacy/releng_7_fast_net/sys: netinet sys X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 01 Mar 2009 05:09:35 -0000 Author: kmacy Date: Sun Mar 1 05:09:34 2009 New Revision: 189206 URL: http://svn.freebsd.org/changeset/base/189206 Log: multiq step 5: flowid: allow TCP to track driver furnished flow identifier and pass it down to the driver in ip_output to give the driver the option of identifying flows to provide connection ordering while using multiple transmit queues Modified: user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c user/kmacy/releng_7_fast_net/sys/netinet/tcp_input.c user/kmacy/releng_7_fast_net/sys/sys/mbuf.h Modified: user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Sun Mar 1 04:59:54 2009 (r189205) +++ user/kmacy/releng_7_fast_net/sys/netinet/in_pcb.h Sun Mar 1 05:09:34 2009 (r189206) @@ -160,7 +160,8 @@ struct inpcb { u_char inp_ip_ttl; /* (i) time to live proto */ u_char inp_ip_p; /* (c) protocol proto */ u_char inp_ip_minttl; /* (i) minimum TTL or drop */ - u_int inp_refcount; /* (i) refcount */ + uint16_t inp_connid; /* (x) connection id / queue id */ + uint16_t inp_refcount; /* (i) refcount */ struct ucred *inp_cred; /* (c) cache of socket cred */ void *inp_pspare[1]; /* L2 information */ struct rtentry *inp_rt; /* L3 information */ Modified: user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c Sun Mar 1 04:59:54 2009 (r189205) +++ user/kmacy/releng_7_fast_net/sys/netinet/ip_output.c Sun Mar 1 05:09:34 2009 (r189206) @@ -129,6 +129,7 @@ ip_output(struct mbuf *m, struct mbuf *o if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); + M_SETFLOWID(m, inp->inp_connid); INP_LOCK_ASSERT(inp); if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) { if (inp->inp_rt->rt_flags & RTF_UP) { Modified: user/kmacy/releng_7_fast_net/sys/netinet/tcp_input.c ============================================================================== --- user/kmacy/releng_7_fast_net/sys/netinet/tcp_input.c Sun Mar 1 04:59:54 2009 (r189205) +++ user/kmacy/releng_7_fast_net/sys/netinet/tcp_input.c Sun Mar 1 05:09:34 2009 (r189206) @@ -478,7 +478,9 @@ findpcb: goto dropwithreset; } INP_WLOCK(inp); - + if (inp->inp_connid == 0) + inp->inp_connid = M_GETFLOWID(m); + #ifdef IPSEC #ifdef INET6 if (isipv6 && ipsec6_in_reject(m, inp)) { Modified: user/kmacy/releng_7_fast_net/sys/sys/mbuf.h ============================================================================== --- user/kmacy/releng_7_fast_net/sys/sys/mbuf.h Sun Mar 1 04:59:54 2009 (r189205) +++ user/kmacy/releng_7_fast_net/sys/sys/mbuf.h Sun Mar 1 05:09:34 2009 (r189206) @@ -962,15 +962,25 @@ m_tag_find(struct mbuf *m, int type, str #define M_FIBSHIFT 28 #define M_FIBMASK 0x0F +#define M_FLOWIDSHIFT 16 + /* get the fib from an mbuf and if it is not set, return the default */ #define M_GETFIB(_m) \ ((((_m)->m_flags & M_FIB) >> M_FIBSHIFT) & M_FIBMASK) #define M_SETFIB(_m, _fib) do { \ _m->m_flags &= ~M_FIB; \ - _m->m_flags |= (((_fib) << M_FIBSHIFT) & M_FIB); \ + _m->m_flags |= (((_fib) << M_FIBSHIFT) & M_FIB); \ } while (0) +#define M_GETFLOWID(_m) \ + ((_m)->m_pkthdr.csum_flags >> M_FLOWIDSHIFT) + +#define M_SETFLOWID(_m, _flowid) do { \ + _m->m_pkthdr.csum_flags |= ((_flowid) << M_FLOWIDSHIFT); \ +} while (0) + + #endif /* _KERNEL */ #endif /* !_SYS_MBUF_H_ */ From owner-svn-src-user@FreeBSD.ORG Tue Mar 3 15:43:34 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 6B163106566B; Tue, 3 Mar 2009 15:43:34 +0000 (UTC) (envelope-from dfr@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 5A82F8FC14; Tue, 3 Mar 2009 15:43:34 +0000 (UTC) (envelope-from dfr@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n23FhYY0014445; Tue, 3 Mar 2009 15:43:34 GMT (envelope-from dfr@svn.freebsd.org) Received: (from dfr@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n23FhYW6014444; Tue, 3 Mar 2009 15:43:34 GMT (envelope-from dfr@svn.freebsd.org) Message-Id: <200903031543.n23FhYW6014444@svn.freebsd.org> From: Doug Rabson Date: Tue, 3 Mar 2009 15:43:34 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189304 - user/dfr/xenhvm/6/sys/amd64/include/xen X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 03 Mar 2009 15:43:34 -0000 Author: dfr Date: Tue Mar 3 15:43:34 2009 New Revision: 189304 URL: http://svn.freebsd.org/changeset/base/189304 Log: Add MFNTOPFN for HVM builds. Modified: user/dfr/xenhvm/6/sys/amd64/include/xen/xenvar.h Modified: user/dfr/xenhvm/6/sys/amd64/include/xen/xenvar.h ============================================================================== --- user/dfr/xenhvm/6/sys/amd64/include/xen/xenvar.h Tue Mar 3 13:12:12 2009 (r189303) +++ user/dfr/xenhvm/6/sys/amd64/include/xen/xenvar.h Tue Mar 3 15:43:34 2009 (r189304) @@ -68,6 +68,7 @@ machtophys(vm_paddr_t ma) #define vtomach(va) pmap_kextract((vm_offset_t) (va)) #define PFNTOMFN(pa) (pa) +#define MFNTOPFN(ma) (ma) #define set_phys_to_machine(pfn, mfn) ((void)0) #define PT_UPDATES_FLUSH() ((void)0) From owner-svn-src-user@FreeBSD.ORG Tue Mar 3 21:45:48 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 159B3106564A; Tue, 3 Mar 2009 21:45:48 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id DDB838FC1D; Tue, 3 Mar 2009 21:45:47 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n23LjloI022264; Tue, 3 Mar 2009 21:45:47 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n23LjlHk022263; Tue, 3 Mar 2009 21:45:47 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903032145.n23LjlHk022263@svn.freebsd.org> From: Kip Macy Date: Tue, 3 Mar 2009 21:45:47 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189324 - user/kmacy/HEAD_fast_net_merge X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 03 Mar 2009 21:45:48 -0000 Author: kmacy Date: Tue Mar 3 21:45:47 2009 New Revision: 189324 URL: http://svn.freebsd.org/changeset/base/189324 Log: create branch for incrementally merging HEAD_fast_net work in to HEAD Added: user/kmacy/HEAD_fast_net_merge/ (props changed) - copied from r189323, head/ From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 00:56:49 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 91A42106566C; Wed, 4 Mar 2009 00:56:49 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 7F0798FC0A; Wed, 4 Mar 2009 00:56:49 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n240un58026166; Wed, 4 Mar 2009 00:56:49 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n240um2r026157; Wed, 4 Mar 2009 00:56:48 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903040056.n240um2r026157@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 00:56:48 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189326 - in user/kmacy/HEAD_fast_net_merge/sys: net netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 00:56:49 -0000 Author: kmacy Date: Wed Mar 4 00:56:48 2009 New Revision: 189326 URL: http://svn.freebsd.org/changeset/base/189326 Log: IFC HEAD_fast_net inpcb rtentry & llentry caching 186573, 186575, 186576, 186578, 186579, 186580 186573: - Cache rtentry in inpcb for connected sockets 186575: - Add cached llentry to inpcb - reduce header pollution by removing net/route.h from in_pcb.h 186576: - install cached llentry in the inpcb and then pass down to ether_output in the struct route 186578: - don't LLE_FREE if llentry is unchanged 186579: - cleanup upgrade / downgrade checks for route and lle 186580: - don't check INP_WLOCKED if inp is NULL Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h user/kmacy/HEAD_fast_net_merge/sys/net/route.h user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 00:56:48 2009 (r189326) @@ -669,6 +669,8 @@ struct ifmultiaddr { }; #ifdef _KERNEL +#define LLTABLE(ifp) ((struct lltable *)(ifp)->if_afdata[AF_INET]) + #define IFAFREE(ifa) \ do { \ IFA_LOCK(ifa); \ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/route.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 00:56:48 2009 (r189326) @@ -46,8 +46,9 @@ * in their control blocks, e.g. inpcb. */ struct route { - struct rtentry *ro_rt; struct sockaddr ro_dst; + struct rtentry *ro_rt; + struct llentry *ro_lle; }; /* Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c Wed Mar 4 00:56:48 2009 (r189326) @@ -81,7 +81,6 @@ __FBSDID("$FreeBSD$"); #define SIN(s) ((struct sockaddr_in *)s) #define SDL(s) ((struct sockaddr_dl *)s) -#define LLTABLE(ifp) ((struct lltable *)(ifp)->if_afdata[AF_INET]) SYSCTL_DECL(_net_link_ether); SYSCTL_NODE(_net_link_ether, PF_INET, inet, CTLFLAG_RW, 0, ""); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 00:56:48 2009 (r189326) @@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include @@ -497,6 +498,72 @@ in_pcbbind_setup(struct inpcb *inp, stru return (0); } +void +in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro) +{ + struct sockaddr_in *sin; + struct sockaddr *dst; + struct llentry *la; + struct rtentry *rt; + struct ifnet *ifp; + int flags = LLE_EXCLUSIVE; + + INP_WLOCK_ASSERT(inp); + bzero(sro, sizeof(*sro)); + sin = (struct sockaddr_in *)&sro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(struct sockaddr_in); + sin->sin_addr.s_addr = faddr; + /* + * If route is known our src addr is taken from the i/f, + * else punt. + * + * Find out route to destination. + */ + if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) { +#ifdef RADIX_MPATH + rtalloc_mpath_fib(sro, ntohl(faddr->s_addr), + inp->inp_inc.inc_fibnum); +#else + in_rtalloc_ign(sro, 0, inp->inp_inc.inc_fibnum); +#endif + } + + rt = sro->ro_rt; + if (rt == NULL) + return; + + inp->inp_rt = rt; + inp->inp_vflag |= INP_RT_VALID; + + if (rt->rt_ifp == NULL) + return; + + ifp = rt->rt_ifp; + dst = &sro->ro_dst; + if (rt->rt_flags & RTF_GATEWAY) + dst = rt->rt_gateway; + + IF_AFDATA_RLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), flags, dst); + IF_AFDATA_RUNLOCK(ifp); + if ((la == NULL) && + (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { + flags |= (LLE_CREATE | LLE_EXCLUSIVE); + IF_AFDATA_WLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), flags, dst); + IF_AFDATA_WUNLOCK(ifp); + } + if (la == NULL) + return; + + LLE_ADDREF(la); + LLE_WUNLOCK(la); + + inp->inp_lle = la; + inp->inp_flags |= INP_LLE_VALID; +} + /* * Connect from a socket to a specified address. * Both address and port must be specified in argument sin. @@ -509,6 +576,7 @@ in_pcbconnect(struct inpcb *inp, struct u_short lport, fport; in_addr_t laddr, faddr; int anonport, error; + struct route sro; INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo); INP_WLOCK_ASSERT(inp); @@ -532,6 +600,7 @@ in_pcbconnect(struct inpcb *inp, struct } } + in_pcbrtalloc(inp, faddr, &sro); /* Commit the remaining changes. */ inp->inp_lport = lport; inp->inp_laddr.s_addr = laddr; @@ -870,6 +939,17 @@ in_pcbdisconnect(struct inpcb *inp) INP_INFO_WLOCK_ASSERT(inp->inp_pcbinfo); INP_WLOCK_ASSERT(inp); + if (inp->inp_vflag & INP_RT_VALID) { + inp->inp_vflag &= ~INP_RT_VALID; + RTFREE(inp->inp_rt); + inp->inp_rt = NULL; + } + if (inp->inp_flags & INP_LLE_VALID) { + inp->inp_flags &= ~INP_LLE_VALID; + LLE_FREE(inp->inp_lle); + inp->inp_lle = NULL; + } + inp->inp_faddr.s_addr = INADDR_ANY; inp->inp_fport = 0; in_pcbrehash(inp); @@ -907,6 +987,17 @@ in_pcbfree_internal(struct inpcb *inp) INP_INFO_WLOCK_ASSERT(ipi); INP_WLOCK_ASSERT(inp); + if (inp->inp_vflag & INP_RT_VALID) { + inp->inp_vflag &= ~INP_RT_VALID; + RTFREE(inp->inp_rt); + inp->inp_rt = NULL; + } + if (inp->inp_flags & INP_LLE_VALID) { + inp->inp_flags &= ~INP_LLE_VALID; + LLE_FREE(inp->inp_lle); + inp->inp_lle = NULL; + } + #ifdef IPSEC if (inp->inp_sp != NULL) ipsec_delete_pcbpolicy(inp); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 00:56:48 2009 (r189326) @@ -39,8 +39,6 @@ #include #include -#include - #ifdef _KERNEL #include #endif @@ -49,6 +47,10 @@ #define in6p_sp inp_sp /* for KAME src sync over BSD*'s */ struct inpcbpolicy; +struct rtentry; +struct llentry; +struct route; + /* * struct inpcb is the common protocol control block structure used in most * IP transport protocols. @@ -169,7 +171,8 @@ struct inpcb { u_char inp_ip_minttl; /* (i) minimum TTL or drop */ uint32_t inp_ispare1; /* (x) connection id / queue id */ u_int inp_refcount; /* (i) refcount */ - void *inp_pspare[2]; /* (x) rtentry / general use */ + struct llentry *inp_lle; /* L2 information */ + struct rtentry *inp_rt; /* L3 information */ /* Local and foreign ports, local and foreign addr. */ struct in_conninfo inp_inc; /* (i/p) list for PCB's local port */ @@ -324,6 +327,9 @@ struct inpcbinfo { #define INP_RLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_RLOCKED) #define INP_WLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_WLOCKED) #define INP_UNLOCK_ASSERT(inp) rw_assert(&(inp)->inp_lock, RA_UNLOCKED) +#define INP_TRY_UPGRADE(inp) rw_try_upgrade(&(inp)->inp_lock) +#define INP_DOWNGRADE(inp) rw_downgrade(&(inp)->inp_lock) +#define INP_WLOCKED(inp) rw_wowned(&(inp)->inp_lock) #ifdef _KERNEL /* @@ -395,6 +401,7 @@ void inp_4tuple_get(struct inpcb *inp, #define INP_ONESBCAST 0x10 /* send all-ones broadcast */ #define INP_DROPPED 0x20 /* protocol drop flag */ #define INP_SOCKREF 0x40 /* strong socket reference */ +#define INP_RT_VALID 0x80 /* rtentry is set */ /* * Flags for inp_flag. @@ -412,7 +419,7 @@ void inp_4tuple_get(struct inpcb *inp, #define INP_RECVTTL 0x400 /* receive incoming IP TTL */ #define INP_DONTFRAG 0x800 /* don't fragment packet */ #define INP_NONLOCALOK 0x1000 /* Allow bind to spoof any address */ - /* - requires options IP_NONLOCALBIND */ +#define INP_LLE_VALID 0x2000 /* L2 entry is set */ #define IN6P_IPV6_V6ONLY 0x008000 /* restrict AF_INET6 socket for v6 */ @@ -489,6 +496,7 @@ void in_pcbdisconnect(struct inpcb *); void in_pcbdrop(struct inpcb *); void in_pcbfree(struct inpcb *); int in_pcbinshash(struct inpcb *); +void in_pcbrtalloc(struct inpcb *inp, in_addr_t faddr, struct route *sro); struct inpcb * in_pcblookup_local(struct inpcbinfo *, struct in_addr, u_short, int, struct ucred *); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 00:56:48 2009 (r189326) @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include @@ -134,11 +135,13 @@ ip_output(struct mbuf *m, struct mbuf *o int hlen = sizeof (struct ip); int mtu; int len, error = 0; + int neednewroute = 0, neednewlle = 0; struct sockaddr_in *dst = NULL; /* keep compiler happy */ struct in_ifaddr *ia = NULL; int isbroadcast, sw_csum; struct route iproute; struct in_addr odst; + struct sockaddr_in *sin; #ifdef IPFIREWALL_FORWARD struct m_tag *fwd_tag = NULL; #endif @@ -152,6 +155,22 @@ ip_output(struct mbuf *m, struct mbuf *o if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); INP_LOCK_ASSERT(inp); + if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) { + if (inp->inp_rt->rt_flags & RTF_UP) { + sin = (struct sockaddr_in *)&ro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(struct sockaddr_in); + sin->sin_addr.s_addr = inp->inp_faddr.s_addr; + ro->ro_rt = inp->inp_rt; + } else + neednewroute = 1; + } + if ((ro == &iproute) && (inp->inp_flags & INP_LLE_VALID)) { + if (inp->inp_lle->la_flags & LLE_VALID) { + ro->ro_lle = inp->inp_lle; + } else + neednewlle = 1; + } } if (opt) { @@ -194,7 +213,8 @@ again: if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 || dst->sin_family != AF_INET || dst->sin_addr.s_addr != ip->ip_dst.s_addr)) { - RTFREE(ro->ro_rt); + if (inp == NULL || (ro->ro_rt != inp->inp_rt)) + RTFREE(ro->ro_rt); ro->ro_rt = (struct rtentry *)NULL; } #ifdef IPFIREWALL_FORWARD @@ -637,9 +657,48 @@ passout: V_ipstat.ips_fragmented++; done: - if (ro == &iproute && ro->ro_rt) { - RTFREE(ro->ro_rt); + if (ro == &iproute && ro->ro_rt != NULL) { + int wlocked = 0; + struct llentry *la; + + if (neednewlle || neednewroute) { + wlocked = INP_WLOCKED(inp); + if (!wlocked && INP_TRY_UPGRADE(inp) == 0) + return (error); + } + + if (inp == NULL || (inp->inp_vflag & INP_RT_VALID) == 0) + RTFREE(ro->ro_rt); + else if (neednewroute && ro->ro_rt != inp->inp_rt) { + RTFREE(inp->inp_rt); + inp->inp_rt = ro->ro_rt; + + } + if (neednewlle) { + IF_AFDATA_RLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, + (struct sockaddr *)dst); + IF_AFDATA_RUNLOCK(ifp); + if ((la == NULL) && + (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { + IF_AFDATA_WLOCK(ifp); + la = lla_lookup(LLTABLE(ifp), + (LLE_CREATE | LLE_EXCLUSIVE), + (struct sockaddr *)dst); + IF_AFDATA_WUNLOCK(ifp); + } + if (la != NULL && (inp->inp_lle != la)) { + LLE_FREE(inp->inp_lle); + LLE_ADDREF(la); + LLE_WUNLOCK(la); + inp->inp_lle = la; + } else if (la != NULL) + LLE_WUNLOCK(la); + } + if ((neednewlle || neednewroute) && !wlocked) + INP_DOWNGRADE(inp); } + return (error); bad: m_freem(m); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_hostcache.c Wed Mar 4 00:56:48 2009 (r189326) @@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_offload.c Wed Mar 4 00:56:48 2009 (r189326) @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Wed Mar 4 00:05:40 2009 (r189325) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_usrreq.c Wed Mar 4 00:56:48 2009 (r189326) @@ -1075,6 +1075,7 @@ tcp_connect(struct tcpcb *tp, struct soc struct socket *so = inp->inp_socket; INIT_VNET_INET(so->so_vnet); struct in_addr laddr; + struct route sro; u_short lport; int error; @@ -1103,6 +1104,7 @@ tcp_connect(struct tcpcb *tp, struct soc inp->inp_laddr = laddr; in_pcbrehash(inp); + in_pcbrtalloc(inp, inp->inp_faddr.s_addr, &sro); /* * Compute window scaling to request: * Scale to fit into sweet spot. See tcp_syncache.c. From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 01:10:39 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 0593A1065677; Wed, 4 Mar 2009 01:10:39 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id E6D768FC1C; Wed, 4 Mar 2009 01:10:38 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n241AcoD026561; Wed, 4 Mar 2009 01:10:38 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n241AcXR026555; Wed, 4 Mar 2009 01:10:38 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903040110.n241AcXR026555@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 01:10:38 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189329 - in user/kmacy/HEAD_fast_net_merge/sys: conf net netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 01:10:39 -0000 Author: kmacy Date: Wed Mar 4 01:10:38 2009 New Revision: 189329 URL: http://svn.freebsd.org/changeset/base/189329 Log: IFC HEAD_fast_net add flowtable to bypass L3 and L2 lookups: 186583: import updated flowtable for use by forwarding and unconnected sockets Added: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.h Modified: user/kmacy/HEAD_fast_net_merge/sys/conf/files user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c user/kmacy/HEAD_fast_net_merge/sys/netinet/vinet.h Modified: user/kmacy/HEAD_fast_net_merge/sys/conf/files ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/conf/files Wed Mar 4 01:01:26 2009 (r189328) +++ user/kmacy/HEAD_fast_net_merge/sys/conf/files Wed Mar 4 01:10:38 2009 (r189329) @@ -2143,6 +2143,7 @@ net/bpf_filter.c optional bpf | netgrap net/bpf_zerocopy.c optional bpf net/bridgestp.c optional bridge | if_bridge net/bsd_comp.c optional ppp_bsdcomp +net/flowtable.c standard net/ieee8023ad_lacp.c optional lagg net/if.c standard net/if_arcsubr.c optional arcnet Added: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 01:10:38 2009 (r189329) @@ -0,0 +1,748 @@ +#include "opt_mpath.h" + +#include +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +#include +#include +#include +#include +#include +#include +#include +#include + +#define calloc(count, size) malloc((count)*(size), M_DEVBUF, M_WAITOK|M_ZERO) + +/* + * Taken from http://burtleburtle.net/bob/c/lookup3.c + */ + +#define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) + +/* +------------------------------------------------------------------------------- +mix -- mix 3 32-bit values reversibly. + +This is reversible, so any information in (a,b,c) before mix() is +still in (a,b,c) after mix(). + +If four pairs of (a,b,c) inputs are run through mix(), or through +mix() in reverse, there are at least 32 bits of the output that +are sometimes the same for one pair and different for another pair. +This was tested for: +* pairs that differed by one bit, by two bits, in any combination + of top bits of (a,b,c), or in any combination of bottom bits of + (a,b,c). +* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + is commonly produced by subtraction) look like a single 1-bit + difference. +* the base values were pseudorandom, all zero but one bit set, or + all zero plus a counter that starts at zero. + +Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that +satisfy this are + 4 6 8 16 19 4 + 9 15 3 18 27 15 + 14 9 3 7 17 3 +Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing +for "differ" defined as + with a one-bit base and a two-bit delta. I +used http://burtleburtle.net/bob/hash/avalanche.html to choose +the operations, constants, and arrangements of the variables. + +This does not achieve avalanche. There are input bits of (a,b,c) +that fail to affect some output bits of (a,b,c), especially of a. The +most thoroughly mixed value is c, but it doesn't really even achieve +avalanche in c. + +This allows some parallelism. Read-after-writes are good at doubling +the number of bits affected, so the goal of mixing pulls in the opposite +direction as the goal of parallelism. I did what I could. Rotates +seem to cost as much as shifts on every machine I could lay my hands +on, and rotates are much kinder to the top and bottom bits, so I used +rotates. +------------------------------------------------------------------------------- +*/ +#define mix(a,b,c) \ +{ \ + a -= c; a ^= rot(c, 4); c += b; \ + b -= a; b ^= rot(a, 6); a += c; \ + c -= b; c ^= rot(b, 8); b += a; \ + a -= c; a ^= rot(c,16); c += b; \ + b -= a; b ^= rot(a,19); a += c; \ + c -= b; c ^= rot(b, 4); b += a; \ +} + +/* +------------------------------------------------------------------------------- +final -- final mixing of 3 32-bit values (a,b,c) into c + +Pairs of (a,b,c) values differing in only a few bits will usually +produce values of c that look totally different. This was tested for +* pairs that differed by one bit, by two bits, in any combination + of top bits of (a,b,c), or in any combination of bottom bits of + (a,b,c). +* "differ" is defined as +, -, ^, or ~^. For + and -, I transformed + the output delta to a Gray code (a^(a>>1)) so a string of 1's (as + is commonly produced by subtraction) look like a single 1-bit + difference. +* the base values were pseudorandom, all zero but one bit set, or + all zero plus a counter that starts at zero. + +These constants passed: + 14 11 25 16 4 14 24 + 12 14 25 16 4 14 24 +and these came close: + 4 8 15 26 3 22 24 + 10 8 15 26 3 22 24 + 11 8 15 26 3 22 24 +------------------------------------------------------------------------------- +*/ +#define final(a,b,c) \ +{ \ + c ^= b; c -= rot(b,14); \ + a ^= c; a -= rot(c,11); \ + b ^= a; b -= rot(a,25); \ + c ^= b; c -= rot(b,16); \ + a ^= c; a -= rot(c,4); \ + b ^= a; b -= rot(a,14); \ + c ^= b; c -= rot(b,24); \ +} + +/* +-------------------------------------------------------------------- + This works on all machines. To be useful, it requires + -- that the key be an array of uint32_t's, and + -- that the length be the number of uint32_t's in the key + + The function hashword() is identical to hashlittle() on little-endian + machines, and identical to hashbig() on big-endian machines, + except that the length has to be measured in uint32_ts rather than in + bytes. hashlittle() is more complicated than hashword() only because + hashlittle() has to dance around fitting the key bytes into registers. +-------------------------------------------------------------------- +*/ +static uint32_t hashword( +const uint32_t *k, /* the key, an array of uint32_t values */ +size_t length, /* the length of the key, in uint32_ts */ +uint32_t initval) /* the previous hash, or an arbitrary value */ +{ + uint32_t a,b,c; + + /* Set up the internal state */ + a = b = c = 0xdeadbeef + (((uint32_t)length)<<2) + initval; + + /*------------------------------------------------- handle most of the key */ + while (length > 3) + { + a += k[0]; + b += k[1]; + c += k[2]; + mix(a,b,c); + length -= 3; + k += 3; + } + + /*------------------------------------------- handle the last 3 uint32_t's */ + switch(length) /* all the case statements fall through */ + { + case 3 : c+=k[2]; + case 2 : b+=k[1]; + case 1 : a+=k[0]; + final(a,b,c); + case 0: /* case 0: nothing left to add */ + break; + } + /*------------------------------------------------------ report the result */ + return c; +} + + +struct ipv4_tuple { + uint16_t ip_sport; /* source port */ + uint16_t ip_dport; /* destination port */ + in_addr_t ip_saddr; /* source address */ + in_addr_t ip_daddr; /* destination address */ +}; + +union ipv4_flow { + struct ipv4_tuple ipf_ipt; + uint32_t ipf_key[3]; +}; + +struct ipv6_tuple { + uint16_t ip_sport; /* source port */ + uint16_t ip_dport; /* destination port */ + struct in6_addr ip_saddr; /* source address */ + struct in6_addr ip_daddr; /* destination address */ +}; + +union ipv6_flow { + struct ipv6_tuple ipf_ipt; + uint32_t ipf_key[9]; +}; + +struct flentry { + volatile uint32_t f_fhash; /* hash flowing forward */ + uint16_t f_flags; /* flow flags */ + uint8_t f_pad; + uint8_t f_proto; /* protocol */ + uint32_t f_uptime; + volatile struct rtentry *f_rt; /* rtentry for flow */ + volatile struct llentry *f_lle; /* llentry for flow */ +}; + +struct flentry_v4 { + struct flentry fl_entry; + union ipv4_flow fl_flow; +}; + +struct flentry_v6 { + struct flentry fl_entry; + union ipv6_flow fl_flow; +}; + +#define fl_fhash fl_entry.fl_fhash +#define fl_flags fl_entry.fl_flags +#define fl_proto fl_entry.fl_proto +#define fl_uptime fl_entry.fl_uptime +#define fl_rt fl_entry.fl_rt +#define fl_lle fl_entry.fl_lle + +#define SECS_PER_HOUR 3600 +#define SECS_PER_DAY (24*SECS_PER_HOUR) + +#define SYN_IDLE 300 +#define UDP_IDLE 300 +#define FIN_WAIT_IDLE 600 +#define TCP_IDLE SECS_PER_DAY + + +typedef void fl_lock_t(struct flowtable *, uint32_t); +typedef void fl_rtalloc_t(struct route *, uint32_t, u_int); + +union flentryp { + struct flentry_v4 *v4; + struct flentry_v6 *v6; + struct flentry_v4 *v4_pcpu[MAXCPU]; + struct flentry_v6 *v6_pcpu[MAXCPU]; +}; + +struct flowtable { + union flentryp ft_table; + int ft_size; + bitstr_t *ft_masks[MAXCPU]; + struct mtx *ft_locks; + int ft_lock_count; + uint32_t ft_flags; + uint32_t ft_collisions; + uint32_t ft_allocated; + uint64_t ft_hits; + + uint32_t ft_udp_idle; + uint32_t ft_fin_wait_idle; + uint32_t ft_syn_idle; + uint32_t ft_tcp_idle; + + fl_lock_t *ft_lock; + fl_lock_t *ft_unlock; + fl_rtalloc_t *ft_rtalloc; + +}; + +static uint32_t hashjitter; + +SYSCTL_NODE(_net_inet, OID_AUTO, flowtable, CTLFLAG_RD, NULL, "flowtable"); + +int flowtable_enable = 1; +SYSCTL_INT(_net_inet_flowtable, OID_AUTO, enable, CTLFLAG_RW, + &flowtable_enable, 0, "enable flowtable caching."); + + +#ifndef RADIX_MPATH +static void +in_rtalloc_ign_wrapper(struct route *ro, uint32_t hash, u_int fib) +{ + + in_rtalloc_ign(ro, 0, fib); +} +#endif + +static void +flowtable_global_lock(struct flowtable *table, uint32_t hash) +{ + int lock_index = (hash)&(table->ft_lock_count - 1); + + mtx_lock(&table->ft_locks[lock_index]); +} + +static void +flowtable_global_unlock(struct flowtable *table, uint32_t hash) +{ + int lock_index = (hash)&(table->ft_lock_count - 1); + + mtx_unlock(&table->ft_locks[lock_index]); +} + +static void +flowtable_pcpu_lock(struct flowtable *table, uint32_t hash) +{ + + critical_enter(); +} + +static void +flowtable_pcpu_unlock(struct flowtable *table, uint32_t hash) +{ + + mb(); + critical_exit(); +} + +#define FL_ENTRY_INDEX(table, hash)((hash) % (table)->ft_size) +#define FL_ENTRY(table, hash) flowtable_entry((table), (hash)) +#define FL_ENTRY_LOCK(table, hash) (table)->ft_lock((table), (hash)) +#define FL_ENTRY_UNLOCK(table, hash) (table)->ft_unlock((table), (hash)) + +#define FL_STALE (1<<8) + +static uint32_t +ipv4_flow_lookup_hash_internal(struct mbuf *m, struct route *ro, + uint32_t *key, uint16_t *flags, uint8_t *protop) +{ + uint16_t sport = 0, dport = 0; + struct ip *ip = mtod(m, struct ip *); + uint8_t proto = ip->ip_p; + int iphlen = ip->ip_hl << 2; + uint32_t hash; + struct sockaddr_in *sin; + struct tcphdr *th; + struct udphdr *uh; + struct sctphdr *sh; + + key[0] = 0; + key[1] = ip->ip_src.s_addr; + key[2] = ip->ip_dst.s_addr; + + sin = (struct sockaddr_in *)&ro->ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(*sin); + sin->sin_addr = ip->ip_dst; + + if (flowtable_enable == 0) + return (0); + + switch (proto) { + case IPPROTO_TCP: + th = (struct tcphdr *)((caddr_t)ip + iphlen); + sport = ntohs(th->th_sport); + dport = ntohs(th->th_dport); + *flags |= th->th_flags; + if (*flags & TH_RST) + *flags |= FL_STALE; + break; + case IPPROTO_UDP: + uh = (struct udphdr *)((caddr_t)ip + iphlen); + sport = uh->uh_sport; + dport = uh->uh_dport; + break; + case IPPROTO_SCTP: + sh = (struct sctphdr *)((caddr_t)ip + iphlen); + sport = sh->src_port; + dport = sh->dest_port; + break; + default: + if (*flags & FL_HASH_PORTS) + goto noop; + /* no port - hence not a protocol we care about */ + break;; + + } + *protop = proto; + + /* + * If this is a transmit route cache then + * hash all flows to a given destination to + * the same bucket + */ + if ((*flags & FL_HASH_PORTS) == 0) + proto = sport = dport = 0; + + ((uint16_t *)key)[0] = sport; + ((uint16_t *)key)[1] = dport; + + hash = hashword(key, 3, hashjitter + proto); + if (m->m_pkthdr.flowid == 0) + m->m_pkthdr.flowid = hash; + + CTR5(KTR_SPARE3, "proto=%d hash=%x key[0]=%x sport=%d dport=%d\n", proto, hash, key[0], sport, dport); + + return (hash); +noop: + *protop = proto; + return (0); +} + +static bitstr_t * +flowtable_mask(struct flowtable *ft) +{ + bitstr_t *mask; + + if (ft->ft_flags & FL_PCPU) + mask = ft->ft_masks[curcpu]; + else + mask = ft->ft_masks[0]; + + return (mask); +} + +static struct flentry * +flowtable_entry(struct flowtable *ft, uint32_t hash) +{ + struct flentry *fle; + int index = (hash % ft->ft_size); + + + if ((ft->ft_flags & FL_IPV6) == 0) { + if (ft->ft_flags & FL_PCPU) { + fle = (struct flentry *) + &ft->ft_table.v4_pcpu[curcpu][index]; + } else + fle = (struct flentry *)&ft->ft_table.v4[index]; + } else { + if (ft->ft_flags & FL_PCPU) + fle = (struct flentry *) + &ft->ft_table.v6_pcpu[curcpu][index]; + else + fle = (struct flentry *)&ft->ft_table.v6[index]; + } + + return (fle); +} + +static int +flow_stale(struct flowtable *ft, struct flentry *fle) +{ + time_t idle_time; + + if ((fle->f_fhash == 0) + || ((fle->f_rt->rt_flags & RTF_HOST) && + ((fle->f_rt->rt_flags & (RTF_UP)) + != (RTF_UP))) + || (fle->f_rt->rt_ifp == NULL)) + return (1); + + idle_time = time_uptime - fle->f_uptime; + + if ((fle->f_flags & FL_STALE) || + ((fle->f_flags & (TH_SYN|TH_ACK|TH_FIN)) == 0 + && (idle_time > ft->ft_udp_idle)) || + ((fle->f_flags & TH_FIN) + && (idle_time > ft->ft_fin_wait_idle)) || + ((fle->f_flags & (TH_SYN|TH_ACK)) == TH_SYN + && (idle_time > ft->ft_syn_idle)) || + ((fle->f_flags & (TH_SYN|TH_ACK)) == (TH_SYN|TH_ACK) + && (idle_time > ft->ft_tcp_idle)) || + ((fle->f_rt->rt_flags & RTF_UP) == 0 || + (fle->f_rt->rt_ifp == NULL))) + return (1); + + return (0); +} + +static void +flowtable_set_hashkey(struct flowtable *ft, struct flentry *fle, uint32_t *key) +{ + uint32_t *hashkey; + int i, nwords; + + if (ft->ft_flags & FL_IPV6) { + nwords = 9; + hashkey = ((struct flentry_v4 *)fle)->fl_flow.ipf_key; + } else { + nwords = 3; + hashkey = ((struct flentry_v6 *)fle)->fl_flow.ipf_key; + } + + for (i = 0; i < nwords; i++) + hashkey[i] = key[i]; +} + +static int +flowtable_insert(struct flowtable *ft, uint32_t hash, uint32_t *key, + uint8_t proto, struct route *ro, uint16_t flags) +{ + struct flentry *fle; + volatile struct rtentry *rt0 = NULL; + struct rtentry *rt1; + int stale; + bitstr_t *mask; + +retry: + FL_ENTRY_LOCK(ft, hash); + mask = flowtable_mask(ft); + fle = flowtable_entry(ft, hash); + if (fle->f_fhash) { + if ((stale = flow_stale(ft, fle)) != 0) { + fle->f_fhash = 0; + rt0 = fle->f_rt; + fle->f_rt = NULL; + bit_clear(mask, FL_ENTRY_INDEX(ft, hash)); + } + FL_ENTRY_UNLOCK(ft, hash); + if (!stale) + return (ENOSPC); + + rt1 = __DEVOLATILE(struct rtentry *, rt0); + RTFREE(rt1); + /* + * We might end up on a different cpu + */ + goto retry; + + } + flowtable_set_hashkey(ft, fle, key); + bit_set(mask, FL_ENTRY_INDEX(ft, hash)); + + fle->f_proto = proto; + fle->f_rt = ro->ro_rt; + fle->f_lle = ro->ro_lle; + fle->f_fhash = hash; + fle->f_uptime = time_uptime; + FL_ENTRY_UNLOCK(ft, hash); + return (0); +} + +static int +flowtable_key_equal(struct flentry *fle, uint32_t *key, int flags) +{ + uint32_t *hashkey; + int i, nwords; + + if (flags & FL_IPV6) { + nwords = 9; + hashkey = ((struct flentry_v4 *)fle)->fl_flow.ipf_key; + } else { + nwords = 3; + hashkey = ((struct flentry_v6 *)fle)->fl_flow.ipf_key; + } + + for (i = 0; i < nwords; i++) + if (hashkey[i] != key[i]) + return (0); + + return (1); +} + +int +flowtable_lookup(struct flowtable *ft, struct mbuf *m, struct route *ro) +{ + uint32_t key[9], hash; + struct flentry *fle; + uint16_t flags; + uint8_t proto = 0; + int cache = 1, error = 0; + struct rtentry *rt; + struct llentry *lle; + + flags = ft ? ft->ft_flags : 0; + ro->ro_rt = NULL; + ro->ro_lle = NULL; + + /* + * The internal hash lookup is the only IPv4 specific bit + * remaining + */ + hash = ipv4_flow_lookup_hash_internal(m, ro, key, + &flags, &proto); + + /* + * Ports are zero and this isn't a transmit cache + * - thus not a protocol for which we need to keep + * statex + * FL_HASH_PORTS => key[0] != 0 for TCP || UDP || SCTP + */ + if (hash == 0 || (key[0] == 0 && (ft->ft_flags & FL_HASH_PORTS))) { + cache = 0; + goto uncached; + } + FL_ENTRY_LOCK(ft, hash); + fle = FL_ENTRY(ft, hash); + rt = __DEVOLATILE(struct rtentry *, fle->f_rt); + lle = __DEVOLATILE(struct llentry *, fle->f_lle); + if ((rt != NULL) + && fle->f_fhash == hash + && flowtable_key_equal(fle, key, flags) + && (proto == fle->f_proto) + && (rt->rt_flags & RTF_UP) + && (rt->rt_ifp != NULL)) { + fle->f_uptime = time_uptime; + fle->f_flags |= flags; + ro->ro_rt = rt; + ro->ro_lle = lle; + FL_ENTRY_UNLOCK(ft, hash); + return (0); + } + FL_ENTRY_UNLOCK(ft, hash); + +uncached: + /* + * This bit of code ends up locking the + * same route 3 times (just like ip_output + ether_output) + * - at lookup + * - in rt_check when called by arpresolve + * - dropping the refcount for the rtentry + * + * This could be consolidated to one if we wrote a variant + * of arpresolve with an rt_check variant that expected to + * receive the route locked + */ + ft->ft_rtalloc(ro, hash, M_GETFIB(m)); + if (ro->ro_rt == NULL) + error = ENETUNREACH; + else { + int finsert; + struct llentry *lle = NULL; + struct sockaddr *l3addr; + struct rtentry *rt = ro->ro_rt; + struct ifnet *ifp = rt->rt_ifp; + + if (rt->rt_flags & RTF_GATEWAY) + l3addr = rt->rt_gateway; + else + l3addr = &ro->ro_dst; + IF_AFDATA_RLOCK(ifp); + lle = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, l3addr); + IF_AFDATA_RUNLOCK(ifp); + if ((lle == NULL) && + (ifp->if_flags & (IFF_NOARP | IFF_STATICARP)) == 0) { + IF_AFDATA_WLOCK(ifp); + lle = lla_lookup(LLTABLE(ifp), + (LLE_CREATE | LLE_EXCLUSIVE), l3addr); + IF_AFDATA_WUNLOCK(ifp); + } + if (lle != NULL) { + LLE_ADDREF(lle); + LLE_WUNLOCK(lle); + } + ro->ro_lle = lle; + finsert = ((lle != NULL) && cache); + if (finsert) + error = flowtable_insert(ft, hash, key, proto, + ro, flags); + + if (error || !finsert) { + RTFREE(rt); + if (lle != NULL) + LLE_FREE(lle); + } + error = 0; + } + + return (error); +} + +#ifdef notyet +static __inline int +bit_fns(bitstr_t *name, int nbits, int lastbit) +{ + int lastbit_start = lastbit & ~0x7; + bitstr_t *bitstr_start = &name[lastbit_start]; + int value = 0; + + while (value <= lastbit && value != 1) + bit_ffs(bitstr_start, nbits, &value); + + return (value); +} +#endif + +struct flowtable * +flowtable_alloc(int nentry, int flags) +{ + struct flowtable *ft; + int i; + + if (hashjitter == 0) + hashjitter = arc4random(); + + ft = malloc(sizeof(struct flowtable), + M_RTABLE, M_WAITOK | M_ZERO); + + ft->ft_flags = flags; + ft->ft_size = nentry; +#ifdef RADIX_MPATH + ft->ft_rtalloc = rtalloc_mpath_fib; +#else + ft->ft_rtalloc = in_rtalloc_ign_wrapper; +#endif + if (flags & FL_PCPU) { + ft->ft_lock = flowtable_pcpu_lock; + ft->ft_unlock = flowtable_pcpu_unlock; + + for (i = 0; i < mp_ncpus; i++) { + ft->ft_table.v4_pcpu[i] = + malloc(nentry*sizeof(struct flentry_v4), + M_RTABLE, M_WAITOK | M_ZERO); + ft->ft_masks[i] = bit_alloc(nentry); + } + } else { + ft->ft_lock_count = 2*(powerof2(mp_ncpus) ? mp_ncpus : + (fls(mp_ncpus) << 1)); + + ft->ft_lock = flowtable_global_lock; + ft->ft_unlock = flowtable_global_unlock; + ft->ft_table.v4 = + malloc(nentry*sizeof(struct flentry_v4), + M_RTABLE, M_WAITOK | M_ZERO); + ft->ft_locks = malloc(ft->ft_lock_count*sizeof(struct mtx), + M_RTABLE, M_WAITOK | M_ZERO); + for (i = 0; i < ft->ft_lock_count; i++) + mtx_init(&ft->ft_locks[i], "flow", NULL, MTX_DEF|MTX_DUPOK); + + ft->ft_masks[0] = bit_alloc(nentry); + } + + /* + * In the local transmit case the table truly is + * just a cache - so everything is eligible for + * replacement after 5s of non-use + */ + if (flags & FL_HASH_PORTS) { + ft->ft_udp_idle = UDP_IDLE; + ft->ft_syn_idle = SYN_IDLE; + ft->ft_fin_wait_idle = FIN_WAIT_IDLE; + ft->ft_tcp_idle = TCP_IDLE; + } else { + ft->ft_udp_idle = ft->ft_fin_wait_idle = + ft->ft_syn_idle = ft->ft_tcp_idle = 30; + + } + + + return (ft); +} + Added: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.h ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.h Wed Mar 4 01:10:38 2009 (r189329) @@ -0,0 +1,24 @@ +#ifndef _NET_FLOWTABLE_H_ +#define _NET_FLOWTABLE_H_ + +#ifdef _KERNEL +#include +#include + +#define FL_HASH_PORTS (1<<0) /* hash 4-tuple + protocol */ +#define FL_PCPU (1<<1) /* pcpu cache */ +#define FL_IPV6 (1<<2) /* IPv6 table */ + +struct flowtable; +struct flowtable *flowtable_alloc(int nentry, int flags); + +/* + * Given a flow table, look up the L3 and L2 information and + * return it in the route + * + */ +int flowtable_lookup(struct flowtable *ft, struct mbuf *m, + struct route *ro); + +#endif +#endif Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Wed Mar 4 01:01:26 2009 (r189328) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Wed Mar 4 01:10:38 2009 (r189329) @@ -64,6 +64,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -211,6 +212,21 @@ SYSCTL_INT(_net_inet_ip, IPCTL_DEFMTU, m SYSCTL_V_INT(V_NET, vnet_inet, _net_inet_ip, OID_AUTO, stealth, CTLFLAG_RW, ipstealth, 0, "IP stealth mode, no TTL decrementation on forwarding"); #endif +static int ip_pcpu_flowtable_size = 2048; +TUNABLE_INT("net.inet.ip.pcpu_flowtable_size", &ip_pcpu_flowtable_size); +SYSCTL_V_INT(V_NET, vnet_inet, _net_inet_ip, OID_AUTO, pcpu_flowtable_size, + CTLFLAG_RDTUN, ip_pcpu_flowtable_size, 0, + "number of entries in the per cpu flow caches"); + +#ifdef RADIX_MPATH +static int ip_global_flowtable_size = 128*1024; +#else +static int ip_global_flowtable_size = 16*1024; +#endif +TUNABLE_INT("net.inet.ip.global_flowtable_size", &ip_global_flowtable_size); +SYSCTL_V_INT(V_NET, vnet_inet, _net_inet_ip, OID_AUTO, global_flowtable_size, + CTLFLAG_RDTUN, ip_global_flowtable_size, 0, + "number of entries in the global flow cache"); /* * ipfw_ether and ipfw_bridge hooks. @@ -221,6 +237,8 @@ ip_dn_io_t *ip_dn_io_ptr = NULL; #ifdef VIMAGE_GLOBALS int fw_one_pass; #endif +struct flowtable *ipv4_ft; +struct flowtable *ipv4_forward_ft; static void ip_freef(struct ipqhead *, struct ipq *); @@ -320,6 +338,9 @@ ip_init(void) ipintrq.ifq_maxlen = ipqmaxlen; mtx_init(&ipintrq.ifq_mtx, "ip_inq", NULL, MTX_DEF); netisr_register(NETISR_IP, ip_input, &ipintrq, 0); + + ipv4_ft = flowtable_alloc(ip_pcpu_flowtable_size, FL_PCPU); + ipv4_forward_ft = flowtable_alloc(ip_global_flowtable_size, FL_HASH_PORTS); } void Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 01:01:26 2009 (r189328) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 01:10:38 2009 (r189329) @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #ifdef RADIX_MPATH #include #endif @@ -114,6 +115,7 @@ static void ip_mloopback extern struct protosw inetsw[]; +extern struct flowtable *ipv4_ft; /* * IP output. The packet in mbuf chain m contains a skeletal IP @@ -135,7 +137,7 @@ ip_output(struct mbuf *m, struct mbuf *o int hlen = sizeof (struct ip); int mtu; int len, error = 0; - int neednewroute = 0, neednewlle = 0; + int neednewroute = 0, neednewlle = 0, nortfree = 0; struct sockaddr_in *dst = NULL; /* keep compiler happy */ struct in_ifaddr *ia = NULL; int isbroadcast, sw_csum; @@ -172,6 +174,11 @@ ip_output(struct mbuf *m, struct mbuf *o neednewlle = 1; } } + if ((ro == &iproute) && (ro->ro_rt == NULL) && (ro->ro_lle == NULL)) { + if (flowtable_lookup(ipv4_ft, m, ro) == 0) + nortfree = 1; + } + if (opt) { len = 0; @@ -213,7 +220,8 @@ again: if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 || dst->sin_family != AF_INET || dst->sin_addr.s_addr != ip->ip_dst.s_addr)) { - if (inp == NULL || (ro->ro_rt != inp->inp_rt)) + if ((nortfree == 0) && + (inp == NULL || (ro->ro_rt != inp->inp_rt))) RTFREE(ro->ro_rt); ro->ro_rt = (struct rtentry *)NULL; } @@ -667,12 +675,12 @@ done: return (error); } - if (inp == NULL || (inp->inp_vflag & INP_RT_VALID) == 0) + if ((nortfree == 0) && + (inp == NULL || (inp->inp_vflag & INP_RT_VALID) == 0)) RTFREE(ro->ro_rt); else if (neednewroute && ro->ro_rt != inp->inp_rt) { RTFREE(inp->inp_rt); inp->inp_rt = ro->ro_rt; - } if (neednewlle) { IF_AFDATA_RLOCK(ifp); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/vinet.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/vinet.h Wed Mar 4 01:01:26 2009 (r189328) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/vinet.h Wed Mar 4 01:10:38 2009 (r189329) @@ -72,6 +72,8 @@ struct vnet_inet { int _ip_sendsourcequench; int _ip_do_randomid; int _ip_checkinterface; + int _ip_pcpu_flowtable_size; + int _ip_global_flowtable_size; u_short _ip_id; uma_zone_t _ipq_zone; From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 01:33:17 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 3B32A106564A; Wed, 4 Mar 2009 01:33:17 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 1F4228FC16; Wed, 4 Mar 2009 01:33:17 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n241XHNm027100; Wed, 4 Mar 2009 01:33:17 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n241XGdk027097; Wed, 4 Mar 2009 01:33:16 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903040133.n241XGdk027097@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 01:33:16 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189332 - in user/kmacy/HEAD_fast_net_merge/sys: net netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 01:33:17 -0000 Author: kmacy Date: Wed Mar 4 01:33:16 2009 New Revision: 189332 URL: http://svn.freebsd.org/changeset/base/189332 Log: extend flowtable usage to other areas 186585, 186621, 186629, 186631 186585: - make ipv4 forwarding flowtable aware 186621: - initialize ro_dst at the beginning of ip_forward 186629: - make flowtable_lookup handle case of NULL mbuf 186631: - use flowtable_lookup in in_pcbladdr to avoid contending in calls to in_pcbconnect_setup when the laddr is not bound Modified: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Modified: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 01:31:09 2009 (r189331) +++ user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 01:33:16 2009 (r189332) @@ -334,26 +334,32 @@ ipv4_flow_lookup_hash_internal(struct mb uint32_t *key, uint16_t *flags, uint8_t *protop) { uint16_t sport = 0, dport = 0; - struct ip *ip = mtod(m, struct ip *); - uint8_t proto = ip->ip_p; - int iphlen = ip->ip_hl << 2; + struct ip *ip; + uint8_t proto = 0; + int iphlen; uint32_t hash; struct sockaddr_in *sin; struct tcphdr *th; struct udphdr *uh; struct sctphdr *sh; - key[0] = 0; - key[1] = ip->ip_src.s_addr; - key[2] = ip->ip_dst.s_addr; + if (flowtable_enable == 0) + return (0); sin = (struct sockaddr_in *)&ro->ro_dst; - sin->sin_family = AF_INET; - sin->sin_len = sizeof(*sin); - sin->sin_addr = ip->ip_dst; + KASSERT(sin->sin_family == AF_INET, + ("bad address passed")); + key[0] = 0; + key[1] = 0; + key[2] = sin->sin_addr.s_addr; - if (flowtable_enable == 0) - return (0); + if (m == NULL || (*flags & FL_HASH_PORTS) == 0) + goto skipports; + + ip = mtod(m, struct ip *); + proto = ip->ip_p; + iphlen = ip->ip_hl << 2; /* XXX options? */ + key[1] = ip->ip_src.s_addr; switch (proto) { case IPPROTO_TCP: @@ -394,8 +400,13 @@ ipv4_flow_lookup_hash_internal(struct mb ((uint16_t *)key)[0] = sport; ((uint16_t *)key)[1] = dport; +skipports: hash = hashword(key, 3, hashjitter + proto); - if (m->m_pkthdr.flowid == 0) +#ifdef notyet + if (m != NULL && (m->m_flags & M_FLOWID) == 0) +#else + if (m != NULL && m->m_pkthdr.flowid != 0) +#endif m->m_pkthdr.flowid = hash; CTR5(KTR_SPARE3, "proto=%d hash=%x key[0]=%x sport=%d dport=%d\n", proto, hash, key[0], sport, dport); @@ -564,7 +575,7 @@ flowtable_lookup(struct flowtable *ft, s struct flentry *fle; uint16_t flags; uint8_t proto = 0; - int cache = 1, error = 0; + int cache = 1, error = 0, fib = 0; struct rtentry *rt; struct llentry *lle; @@ -620,7 +631,10 @@ uncached: * of arpresolve with an rt_check variant that expected to * receive the route locked */ - ft->ft_rtalloc(ro, hash, M_GETFIB(m)); + if (m != NULL) + fib = M_GETFIB(m); + + ft->ft_rtalloc(ro, hash, fib); if (ro->ro_rt == NULL) error = ENETUNREACH; else { Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 01:31:09 2009 (r189331) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.c Wed Mar 4 01:33:16 2009 (r189332) @@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -88,6 +89,7 @@ __FBSDID("$FreeBSD$"); #include +extern struct flowtable *ipv4_ft; #ifdef VIMAGE_GLOBALS /* * These configure the range of local port addresses assigned to @@ -643,11 +645,14 @@ in_pcbladdr(struct inpcb *inp, struct in * If route is known our src addr is taken from the i/f, * else punt. * + * XXX need to account for fibnum * Find out route to destination. */ if ((inp->inp_socket->so_options & SO_DONTROUTE) == 0) + flowtable_lookup(ipv4_ft, NULL, &sro); +#if 0 in_rtalloc_ign(&sro, 0, inp->inp_inc.inc_fibnum); - +#endif /* * If we found a route, use the address corresponding to * the outgoing interface. @@ -811,8 +816,6 @@ in_pcbladdr(struct inpcb *inp, struct in } done: - if (sro.ro_rt != NULL) - RTFREE(sro.ro_rt); return (error); } Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Wed Mar 4 01:31:09 2009 (r189331) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Wed Mar 4 01:33:16 2009 (r189332) @@ -1349,8 +1349,10 @@ ip_forward(struct mbuf *m, int srcrt) struct in_ifaddr *ia = NULL; struct mbuf *mcopy; struct in_addr dest; + struct sockaddr_in *sin; struct route ro; int error, type = 0, code = 0, mtu = 0; + int flerror; if (m->m_flags & (M_BCAST|M_MCAST) || in_canforward(ip->ip_dst) == 0) { V_ipstat.ips_cantforward++; @@ -1368,8 +1370,18 @@ ip_forward(struct mbuf *m, int srcrt) #ifdef IPSTEALTH } #endif + bzero(&ro, sizeof(ro)); + sin = (struct sockaddr_in *)&ro.ro_dst; + sin->sin_family = AF_INET; + sin->sin_len = sizeof(*sin); + sin->sin_addr = ip->ip_dst; + + flerror = flowtable_lookup(ipv4_forward_ft, m, &ro); + if (flerror == 0) + ia = ifatoia(ro.ro_rt->rt_ifa); + else + ia = ip_rtaddr(ip->ip_dst, M_GETFIB(m)); - ia = ip_rtaddr(ip->ip_dst, M_GETFIB(m)); if (!srcrt && ia == NULL) { icmp_error(m, ICMP_UNREACH, ICMP_UNREACH_HOST, 0, 0); return; @@ -1426,19 +1438,15 @@ ip_forward(struct mbuf *m, int srcrt) */ dest.s_addr = 0; if (!srcrt && V_ipsendredirects && ia->ia_ifp == m->m_pkthdr.rcvif) { - struct sockaddr_in *sin; struct rtentry *rt; - bzero(&ro, sizeof(ro)); - sin = (struct sockaddr_in *)&ro.ro_dst; - sin->sin_family = AF_INET; - sin->sin_len = sizeof(*sin); - sin->sin_addr = ip->ip_dst; - in_rtalloc_ign(&ro, 0, M_GETFIB(m)); + if (flerror != 0) + in_rtalloc_ign(&ro, 0, M_GETFIB(m)); rt = ro.ro_rt; - if (rt && (rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0 && + if (rt != NULL && + (rt->rt_flags & (RTF_DYNAMIC|RTF_MODIFIED)) == 0 && satosin(rt_key(rt))->sin_addr.s_addr != 0) { #define RTA(rt) ((struct in_ifaddr *)(rt->rt_ifa)) u_long src = ntohl(ip->ip_src.s_addr); @@ -1454,7 +1462,7 @@ ip_forward(struct mbuf *m, int srcrt) code = ICMP_REDIRECT_HOST; } } - if (rt) + if (rt && (flerror != 0)) RTFREE(rt); } @@ -1462,13 +1470,14 @@ ip_forward(struct mbuf *m, int srcrt) * Try to cache the route MTU from ip_output so we can consider it for * the ICMP_UNREACH_NEEDFRAG "Next-Hop MTU" field described in RFC1191. */ - bzero(&ro, sizeof(ro)); + if (flerror != 0) + bzero(&ro, sizeof(ro)); error = ip_output(m, NULL, &ro, IP_FORWARDING, NULL, NULL); if (error == EMSGSIZE && ro.ro_rt) mtu = ro.ro_rt->rt_rmx.rmx_mtu; - if (ro.ro_rt) + if (ro.ro_rt && flerror != 0) RTFREE(ro.ro_rt); if (error) From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 01:47:51 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 191B0106566B; Wed, 4 Mar 2009 01:47:51 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 07C928FC24; Wed, 4 Mar 2009 01:47:51 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n241loYw027413; Wed, 4 Mar 2009 01:47:50 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n241loRm027407; Wed, 4 Mar 2009 01:47:50 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903040147.n241loRm027407@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 01:47:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189333 - in user/kmacy/HEAD_fast_net_merge/sys: dev/cxgb net netinet sys X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 01:47:51 -0000 Author: kmacy Date: Wed Mar 4 01:47:50 2009 New Revision: 189333 URL: http://svn.freebsd.org/changeset/base/189333 Log: enable flowid usage 186588, 186591, 186592, 186594 186588: (included) - enabled flowid usage for supporting multiple transmit queues 186591, 186592: (included) - validate so before checking options in tcp_input - so isn't set yet, reference inp_socket 186594: (included) - set flowid valid flag in cxgb Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/cxgb/cxgb_sge.c user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_input.c user/kmacy/HEAD_fast_net_merge/sys/sys/mbuf.h Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/cxgb/cxgb_sge.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/cxgb/cxgb_sge.c Wed Mar 4 01:33:16 2009 (r189332) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/cxgb/cxgb_sge.c Wed Mar 4 01:47:50 2009 (r189333) @@ -2879,6 +2879,7 @@ process_responses(adapter_t *adap, struc eop = get_packet(adap, drop_thresh, qs, &rspq->rspq_mbuf, r); #endif #ifdef IFNET_MULTIQUEUE + rspq->rspq_mh.mh_head->m_flags |= M_FLOWID; rspq->rspq_mh.mh_head->m_pkthdr.flowid = rss_hash; #endif ethpad = 2; Modified: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 01:33:16 2009 (r189332) +++ user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 01:47:50 2009 (r189333) @@ -402,11 +402,7 @@ ipv4_flow_lookup_hash_internal(struct mb skipports: hash = hashword(key, 3, hashjitter + proto); -#ifdef notyet if (m != NULL && (m->m_flags & M_FLOWID) == 0) -#else - if (m != NULL && m->m_pkthdr.flowid != 0) -#endif m->m_pkthdr.flowid = hash; CTR5(KTR_SPARE3, "proto=%d hash=%x key[0]=%x sport=%d dport=%d\n", proto, hash, key[0], sport, dport); Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 01:33:16 2009 (r189332) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/in_pcb.h Wed Mar 4 01:47:50 2009 (r189333) @@ -169,7 +169,7 @@ struct inpcb { u_char inp_ip_ttl; /* (i) time to live proto */ u_char inp_ip_p; /* (c) protocol proto */ u_char inp_ip_minttl; /* (i) minimum TTL or drop */ - uint32_t inp_ispare1; /* (x) connection id / queue id */ + uint32_t inp_flowid; /* (x) flow id / queue id */ u_int inp_refcount; /* (i) refcount */ struct llentry *inp_lle; /* L2 information */ struct rtentry *inp_rt; /* L3 information */ @@ -420,6 +420,8 @@ void inp_4tuple_get(struct inpcb *inp, #define INP_DONTFRAG 0x800 /* don't fragment packet */ #define INP_NONLOCALOK 0x1000 /* Allow bind to spoof any address */ #define INP_LLE_VALID 0x2000 /* L2 entry is set */ +#define INP_SW_FLOWID 0x2000 /* software generated flow id */ +#define INP_HW_FLOWID 0x4000 /* hardware generated flow id */ #define IN6P_IPV6_V6ONLY 0x008000 /* restrict AF_INET6 socket for v6 */ Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 01:33:16 2009 (r189332) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 01:47:50 2009 (r189333) @@ -157,6 +157,10 @@ ip_output(struct mbuf *m, struct mbuf *o if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); INP_LOCK_ASSERT(inp); + if (inp->inp_flags & (INP_HW_FLOWID|INP_SW_FLOWID)) { + m->m_pkthdr.flowid = inp->inp_flowid; + m->m_flags |= M_FLOWID; + } if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) { if (inp->inp_rt->rt_flags & RTF_UP) { sin = (struct sockaddr_in *)&ro->ro_dst; Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_input.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_input.c Wed Mar 4 01:33:16 2009 (r189332) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_input.c Wed Mar 4 01:47:50 2009 (r189333) @@ -594,7 +594,14 @@ findpcb: goto dropwithreset; } INP_WLOCK(inp); - + if (!(inp->inp_flags & INP_HW_FLOWID) + && (m->m_flags & M_FLOWID) + && ((inp->inp_socket == NULL) + || !(inp->inp_socket->so_options & SO_ACCEPTCONN))) { + inp->inp_flags |= INP_HW_FLOWID; + inp->inp_flags &= ~INP_SW_FLOWID; + inp->inp_flowid = m->m_pkthdr.flowid; + } #ifdef IPSEC #ifdef INET6 if (isipv6 && ipsec6_in_reject(m, inp)) { Modified: user/kmacy/HEAD_fast_net_merge/sys/sys/mbuf.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/sys/mbuf.h Wed Mar 4 01:33:16 2009 (r189332) +++ user/kmacy/HEAD_fast_net_merge/sys/sys/mbuf.h Wed Mar 4 01:47:50 2009 (r189333) @@ -195,6 +195,7 @@ struct mbuf { #define M_PROTO6 0x00080000 /* protocol-specific */ #define M_PROTO7 0x00100000 /* protocol-specific */ #define M_PROTO8 0x00200000 /* protocol-specific */ +#define M_FLOWID 0x00400000 /* flowid is valid */ /* * For RELENG_{6,7} steal these flags for limited multiple routing table * support. In RELENG_8 and beyond, use just one flag and a tag. From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 02:38:39 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 2A998106564A; Wed, 4 Mar 2009 02:38:39 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 17F3C8FC0A; Wed, 4 Mar 2009 02:38:39 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n242cdvB028869; Wed, 4 Mar 2009 02:38:39 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n242ccNQ028860; Wed, 4 Mar 2009 02:38:38 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903040238.n242ccNQ028860@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 02:38:38 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189342 - in user/kmacy/HEAD_fast_net_merge: sbin/route sys/net sys/netinet usr.sbin/route6d X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 02:38:39 -0000 Author: kmacy Date: Wed Mar 4 02:38:38 2009 New Revision: 189342 URL: http://svn.freebsd.org/changeset/base/189342 Log: add route weighting and generalizing of affinity to source ip instead of per-flow 186625: - import kernel support for route shutdown 186626: - import user support for route shutdown 186628: - don't lookup laddr or lport if they're already set 186630: - fix route shutdown merge 186923: - Add kernel support for weighting routes 186924: - remove RTA_GENMASK 186925: - remove genmask - add -weight option to route for adding / changing route weight 1868994: - add kernel support for "sticky" routes (all connections from a given source ip will be routed to the same dst ip) 186995: - add support to the route command for making routes sticky 187003: - add new flags to route output - remove hopcount 187004: - update route flags and metricnames in route command 187005: - fetch weight when getting metrics 187006: - try to improve formatting slightly in route 187007: - more output futzing - add show as alias for get 187008: - update show handling 187009: - remove shutdown - update route flags 187010: - fix flag setting in RTM_CHANGE 187011: - add debug cruft to route selection 187012: - fix rn_mpath_count and reduce frequency of printing 187013: - update loop condition print hash earlier 187040: - reduce default timeouts in the flowtable - remove references to shutdown (redundant with zero weight route) - simplify weight checking 187041: - fix radix_mpath comment - remove shutdown flag and message 187206: - include opt_mpath.h so that RADIX_MPATH will be pulled in - remove locking overhead to forwarding workloads by making forwarding table pcpu Modified: user/kmacy/HEAD_fast_net_merge/sbin/route/keywords user/kmacy/HEAD_fast_net_merge/sbin/route/route.c user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c user/kmacy/HEAD_fast_net_merge/sys/net/radix_mpath.c user/kmacy/HEAD_fast_net_merge/sys/net/route.c user/kmacy/HEAD_fast_net_merge/sys/net/route.h user/kmacy/HEAD_fast_net_merge/sys/net/rtsock.c user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c user/kmacy/HEAD_fast_net_merge/usr.sbin/route6d/route6d.c Modified: user/kmacy/HEAD_fast_net_merge/sbin/route/keywords ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sbin/route/keywords Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sbin/route/keywords Wed Mar 4 02:38:38 2009 (r189342) @@ -33,6 +33,7 @@ mtu net netmask nostatic +nostick osi prefixlen proto1 @@ -44,8 +45,11 @@ rtt rttvar sa sendpipe +show ssthresh static +sticky +weight x25 xns xresolve Modified: user/kmacy/HEAD_fast_net_merge/sbin/route/route.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sbin/route/route.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sbin/route/route.c Wed Mar 4 02:38:38 2009 (r189342) @@ -169,6 +169,7 @@ main(argc, argv) if (*argv) switch (keyword(*argv)) { case K_GET: + case K_SHOW: uid = 0; /* FALLTHROUGH */ @@ -548,6 +549,7 @@ set_metric(value, key) caseof(K_SSTHRESH, RTV_SSTHRESH, rmx_ssthresh); caseof(K_RTT, RTV_RTT, rmx_rtt); caseof(K_RTTVAR, RTV_RTTVAR, rmx_rttvar); + caseof(K_WEIGHT, RTV_WEIGHT, rmx_weight); } rtm_inits |= flag; if (lockrest || locking) @@ -571,8 +573,9 @@ newroute(argc, argv) errx(EX_NOPERM, "must be root to alter routing table"); } cmd = argv[0]; - if (*cmd != 'g') + if (*cmd != 'g' && *cmd != 's') shutdown(s, SHUT_RD); /* Don't want to read back our messages */ + while (--argc > 0) { if (**(++argv)== '-') { switch (key = keyword(1 + *argv)) { @@ -635,6 +638,12 @@ newroute(argc, argv) case K_STATIC: flags |= RTF_STATIC; break; + case K_STICKY: + flags |= RTF_STICKY; + break; + case K_NOSTICK: + flags &= ~RTF_STICKY; + break; case K_IFA: if (!--argc) usage((char *)NULL); @@ -645,11 +654,6 @@ newroute(argc, argv) usage((char *)NULL); (void) getaddr(RTA_IFP, *++argv, 0); break; - case K_GENMASK: - if (!--argc) - usage((char *)NULL); - (void) getaddr(RTA_GENMASK, *++argv, 0); - break; case K_GATEWAY: if (!--argc) usage((char *)NULL); @@ -688,6 +692,7 @@ newroute(argc, argv) case K_SSTHRESH: case K_RTT: case K_RTTVAR: + case K_WEIGHT: if (!--argc) usage((char *)NULL); set_metric(*++argv, key); @@ -741,7 +746,7 @@ newroute(argc, argv) } else break; } - if (*cmd == 'g') + if (*cmd == 'g' || *cmd == 's') exit(ret != 0); if (!qflag) { oerrno = errno; @@ -925,9 +930,6 @@ getaddr(which, s, hpp) case RTA_NETMASK: su = &so_mask; break; - case RTA_GENMASK: - su = &so_genmask; - break; case RTA_IFP: su = &so_ifp; afamily = AF_LINK; @@ -1191,7 +1193,7 @@ rtmsg(cmd, flags) cmd = RTM_ADD; else if (cmd == 'c') cmd = RTM_CHANGE; - else if (cmd == 'g') { + else if (cmd == 'g' || cmd == 's') { cmd = RTM_GET; if (so_ifp.sa.sa_family == 0) { so_ifp.sa.sa_family = AF_LINK; @@ -1208,13 +1210,11 @@ rtmsg(cmd, flags) rtm.rtm_addrs = rtm_addrs; rtm.rtm_rmx = rt_metrics; rtm.rtm_inits = rtm_inits; - if (rtm_addrs & RTA_NETMASK) mask_addr(); NEXTADDR(RTA_DST, so_dst); NEXTADDR(RTA_GATEWAY, so_gate); NEXTADDR(RTA_NETMASK, so_mask); - NEXTADDR(RTA_GENMASK, so_genmask); NEXTADDR(RTA_IFP, so_ifp); NEXTADDR(RTA_IFA, so_ifa); rtm.rtm_msglen = l = cp - (char *)&m_rtmsg; @@ -1295,13 +1295,13 @@ char *msgtypes[] = { }; char metricnames[] = -"\011pksent\010rttvar\7rtt\6ssthresh\5sendpipe\4recvpipe\3expire\2hopcount" +"\011weight\010rttvar\7rtt\6ssthresh\5sendpipe\4recvpipe\3expire" "\1mtu"; char routeflags[] = -"\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE\010MASK_PRESENT" -"\011CLONING\012XRESOLVE\013LLINFO\014STATIC\015BLACKHOLE\016b016" -"\017PROTO2\020PROTO1\021PRCLONING\022WASCLONED\023PROTO3\024CHAINDELETE" -"\025PINNED\026LOCAL\027BROADCAST\030MULTICAST"; +"\1UP\2GATEWAY\3HOST\4REJECT\5DYNAMIC\6MODIFIED\7DONE" +"\012XRESOLVE\013LLINFO\014STATIC\015BLACKHOLE" +"\017PROTO2\020PROTO1\021PRCLONING\022WASCLONED\023PROTO3" +"\025PINNED\026LOCAL\027BROADCAST\030MULTICAST\035STICKY"; char ifnetflags[] = "\1UP\2BROADCAST\3DEBUG\4LOOPBACK\5PTP\6b6\7RUNNING\010NOARP" "\011PPROMISC\012ALLMULTI\013OACTIVE\014SIMPLEX\015LINK0\016LINK1" @@ -1464,14 +1464,13 @@ print_getmsg(rtm, msglen) #define msec(u) (((u) + 500) / 1000) /* usec to msec */ (void) printf("\n%s\n", "\ - recvpipe sendpipe ssthresh rtt,msec rttvar hopcount mtu expire"); + recvpipe sendpipe ssthresh rtt,msec mtu weight expire"); printf("%8ld%c ", rtm->rtm_rmx.rmx_recvpipe, lock(RPIPE)); printf("%8ld%c ", rtm->rtm_rmx.rmx_sendpipe, lock(SPIPE)); printf("%8ld%c ", rtm->rtm_rmx.rmx_ssthresh, lock(SSTHRESH)); printf("%8ld%c ", msec(rtm->rtm_rmx.rmx_rtt), lock(RTT)); - printf("%8ld%c ", msec(rtm->rtm_rmx.rmx_rttvar), lock(RTTVAR)); - printf("%8ld%c ", rtm->rtm_rmx.rmx_hopcount, lock(HOPCOUNT)); printf("%8ld%c ", rtm->rtm_rmx.rmx_mtu, lock(MTU)); + printf("%8ld%c ", rtm->rtm_rmx.rmx_weight, lock(WEIGHT)); if (rtm->rtm_rmx.rmx_expire) rtm->rtm_rmx.rmx_expire -= time(0); printf("%8ld%c\n", rtm->rtm_rmx.rmx_expire, lock(EXPIRE)); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sys/net/flowtable.c Wed Mar 4 02:38:38 2009 (r189342) @@ -232,13 +232,10 @@ struct flentry_v6 { #define fl_rt fl_entry.fl_rt #define fl_lle fl_entry.fl_lle -#define SECS_PER_HOUR 3600 -#define SECS_PER_DAY (24*SECS_PER_HOUR) - -#define SYN_IDLE 300 -#define UDP_IDLE 300 -#define FIN_WAIT_IDLE 600 -#define TCP_IDLE SECS_PER_DAY +#define SYN_IDLE 120 +#define UDP_IDLE 60 +#define FIN_WAIT_IDLE 300 +#define TCP_IDLE 1200 typedef void fl_lock_t(struct flowtable *, uint32_t); @@ -331,13 +328,14 @@ flowtable_pcpu_unlock(struct flowtable * static uint32_t ipv4_flow_lookup_hash_internal(struct mbuf *m, struct route *ro, - uint32_t *key, uint16_t *flags, uint8_t *protop) + uint32_t *key, uint16_t *flags, uint8_t *protop, uint32_t *hash, + uint32_t *hash_noports) { uint16_t sport = 0, dport = 0; struct ip *ip; uint8_t proto = 0; int iphlen; - uint32_t hash; + uint32_t rh; struct sockaddr_in *sin; struct tcphdr *th; struct udphdr *uh; @@ -353,14 +351,16 @@ ipv4_flow_lookup_hash_internal(struct mb key[1] = 0; key[2] = sin->sin_addr.s_addr; - if (m == NULL || (*flags & FL_HASH_PORTS) == 0) + if (m == NULL) goto skipports; - ip = mtod(m, struct ip *); proto = ip->ip_p; iphlen = ip->ip_hl << 2; /* XXX options? */ key[1] = ip->ip_src.s_addr; - + + if ((*flags & FL_HASH_PORTS) == 0) + goto skipports; + switch (proto) { case IPPROTO_TCP: th = (struct tcphdr *)((caddr_t)ip + iphlen); @@ -387,30 +387,27 @@ ipv4_flow_lookup_hash_internal(struct mb break;; } - *protop = proto; - - /* - * If this is a transmit route cache then - * hash all flows to a given destination to - * the same bucket - */ - if ((*flags & FL_HASH_PORTS) == 0) - proto = sport = dport = 0; - - ((uint16_t *)key)[0] = sport; - ((uint16_t *)key)[1] = dport; skipports: - hash = hashword(key, 3, hashjitter + proto); + rh = hashword(key, 3, hashjitter + proto); + *hash_noports = rh; + *hash = 0; + if ((*flags & FL_HASH_PORTS) && sport) { + ((uint16_t *)key)[0] = sport; + ((uint16_t *)key)[1] = dport; + rh = hashword(key, 3, hashjitter + proto); + *hash = rh; + } if (m != NULL && (m->m_flags & M_FLOWID) == 0) - m->m_pkthdr.flowid = hash; - - CTR5(KTR_SPARE3, "proto=%d hash=%x key[0]=%x sport=%d dport=%d\n", proto, hash, key[0], sport, dport); - - return (hash); + m->m_pkthdr.flowid = rh; + + CTR5(KTR_SPARE3, "proto=%d hash=%x key[0]=%x sport=%d dport=%d\n", + proto, *hash, key[0], sport, dport); + + return (0); noop: *protop = proto; - return (0); + return (ENOENT); } static bitstr_t * @@ -567,7 +564,7 @@ flowtable_key_equal(struct flentry *fle, int flowtable_lookup(struct flowtable *ft, struct mbuf *m, struct route *ro) { - uint32_t key[9], hash; + uint32_t key[9], hash, hash_noports; struct flentry *fle; uint16_t flags; uint8_t proto = 0; @@ -578,13 +575,14 @@ flowtable_lookup(struct flowtable *ft, s flags = ft ? ft->ft_flags : 0; ro->ro_rt = NULL; ro->ro_lle = NULL; - + hash = hash_noports = 0; + /* * The internal hash lookup is the only IPv4 specific bit * remaining */ - hash = ipv4_flow_lookup_hash_internal(m, ro, key, - &flags, &proto); + error = ipv4_flow_lookup_hash_internal(m, ro, key, + &flags, &proto, &hash, &hash_noports); /* * Ports are zero and this isn't a transmit cache @@ -592,10 +590,13 @@ flowtable_lookup(struct flowtable *ft, s * statex * FL_HASH_PORTS => key[0] != 0 for TCP || UDP || SCTP */ - if (hash == 0 || (key[0] == 0 && (ft->ft_flags & FL_HASH_PORTS))) { + if (error == ENOENT || (key[0] == 0 && (ft->ft_flags & FL_HASH_PORTS))) { cache = 0; goto uncached; } + if ((ft->ft_flags & FL_HASH_PORTS) == 0) + goto skipports; + FL_ENTRY_LOCK(ft, hash); fle = FL_ENTRY(ft, hash); rt = __DEVOLATILE(struct rtentry *, fle->f_rt); @@ -615,6 +616,27 @@ flowtable_lookup(struct flowtable *ft, s } FL_ENTRY_UNLOCK(ft, hash); +skipports: + key[0] = 0; + FL_ENTRY_LOCK(ft, hash_noports); + fle = FL_ENTRY(ft, hash_noports); + rt = __DEVOLATILE(struct rtentry *, fle->f_rt); + lle = __DEVOLATILE(struct llentry *, fle->f_lle); + if ((rt != NULL) + && fle->f_fhash == hash_noports + && flowtable_key_equal(fle, key, flags) + && (proto == fle->f_proto) + && (rt->rt_flags & RTF_UP) + && (rt->rt_ifp != NULL)) { + fle->f_uptime = time_uptime; + fle->f_flags |= flags; + ro->ro_rt = rt; + ro->ro_lle = lle; + FL_ENTRY_UNLOCK(ft, hash_noports); + return (0); + } + FL_ENTRY_UNLOCK(ft, hash_noports); + uncached: /* * This bit of code ends up locking the @@ -640,6 +662,18 @@ uncached: struct rtentry *rt = ro->ro_rt; struct ifnet *ifp = rt->rt_ifp; + if (rt->rt_flags & RTF_STICKY) { + RTFREE(rt); + hash = hash_noports; + ft->ft_rtalloc(ro, hash, fib); + if (ro->ro_rt == NULL) { + error = ENETUNREACH; + goto done; + } + rt = ro->ro_rt; + ifp = rt->rt_ifp; + } + if (rt->rt_flags & RTF_GATEWAY) l3addr = rt->rt_gateway; else @@ -671,7 +705,7 @@ uncached: } error = 0; } - +done: return (error); } @@ -720,7 +754,7 @@ flowtable_alloc(int nentry, int flags) ft->ft_masks[i] = bit_alloc(nentry); } } else { - ft->ft_lock_count = 2*(powerof2(mp_ncpus) ? mp_ncpus : + ft->ft_lock_count = 8*(powerof2(mp_ncpus) ? mp_ncpus : (fls(mp_ncpus) << 1)); ft->ft_lock = flowtable_global_lock; Modified: user/kmacy/HEAD_fast_net_merge/sys/net/radix_mpath.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/radix_mpath.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sys/net/radix_mpath.c Wed Mar 4 02:38:38 2009 (r189342) @@ -77,15 +77,18 @@ rn_mpath_next(struct radix_node *rn) return NULL; } -u_int32_t +uint32_t rn_mpath_count(struct radix_node *rn) { - u_int32_t i; - - i = 1; - while ((rn = rn_mpath_next(rn)) != NULL) - i++; - return i; + uint32_t i = 0; + struct rtentry *rt; + + while (rn != NULL) { + rt = (struct rtentry *)rn; + i += rt->rt_rmx.rmx_weight; + rn = rn_mpath_next(rn); + } + return (i); } struct rtentry * @@ -256,10 +259,12 @@ different: } void -rtalloc_mpath_fib(struct route *ro, u_int32_t hash, u_int fibnum) +rtalloc_mpath_fib(struct route *ro, uint32_t hash, u_int fibnum) { struct radix_node *rn0, *rn; u_int32_t n; + struct rtentry *rt; + int64_t weight; /* * XXX we don't attempt to lookup cached route again; what should @@ -284,25 +289,31 @@ rtalloc_mpath_fib(struct route *ro, u_in /* gw selection by Modulo-N Hash (RFC2991) XXX need improvement? */ hash += hashjitter; hash %= n; - while (hash-- > 0 && rn) { + for (weight = abs((int32_t)hash), rt = ro->ro_rt; + weight >= rt->rt_rmx.rmx_weight && rn; + weight -= rt->rt_rmx.rmx_weight) { + /* stay within the multipath routes */ if (rn->rn_dupedkey && rn->rn_mask != rn->rn_dupedkey->rn_mask) break; rn = rn->rn_dupedkey; + rt = (struct rtentry *)rn; } - /* XXX try filling rt_gwroute and avoid unreachable gw */ - /* if gw selection fails, use the first match (default) */ + /* gw selection has failed - there must be only zero weight routes */ if (!rn) { RT_UNLOCK(ro->ro_rt); + ro->ro_rt = NULL; return; } - - RTFREE_LOCKED(ro->ro_rt); - ro->ro_rt = (struct rtentry *)rn; - RT_LOCK(ro->ro_rt); - RT_ADDREF(ro->ro_rt); + if (ro->ro_rt != rt) { + RTFREE_LOCKED(ro->ro_rt); + ro->ro_rt = (struct rtentry *)rn; + RT_LOCK(ro->ro_rt); + RT_ADDREF(ro->ro_rt); + + } RT_UNLOCK(ro->ro_rt); } Modified: user/kmacy/HEAD_fast_net_merge/sys/net/route.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/route.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sys/net/route.c Wed Mar 4 02:38:38 2009 (r189342) @@ -803,6 +803,103 @@ bad: return (error); } +#ifdef RADIX_MPATH +static int +rn_mpath_update(int req, struct rt_addrinfo *info, + struct radix_node_head *rnh, struct rtentry **ret_nrt) +{ + /* + * if we got multipath routes, we require users to specify + * a matching RTAX_GATEWAY. + */ + struct rtentry *rt, *rto = NULL; + register struct radix_node *rn; + int error = 0; + + rn = rnh->rnh_matchaddr(dst, rnh); + if (rn == NULL) + return (ESRCH); + rto = rt = RNTORT(rn); + rt = rt_mpath_matchgate(rt, gateway); + if (rt == NULL) + return (ESRCH); + /* + * this is the first entry in the chain + */ + if (rto == rt) { + rn = rn_mpath_next((struct radix_node *)rt); + /* + * there is another entry, now it's active + */ + if (rn) { + rto = RNTORT(rn); + RT_LOCK(rto); + rto->rt_flags |= RTF_UP; + RT_UNLOCK(rto); + } else if (rt->rt_flags & RTF_GATEWAY) { + /* + * For gateway routes, we need to + * make sure that we we are deleting + * the correct gateway. + * rt_mpath_matchgate() does not + * check the case when there is only + * one route in the chain. + */ + if (gateway && + (rt->rt_gateway->sa_len != gateway->sa_len || + memcmp(rt->rt_gateway, gateway, gateway->sa_len))) + error = ESRCH; + goto done; + } + /* + * use the normal delete code to remove + * the first entry + */ + if (req != RTM_DELETE) + goto nondelete; + + error = ENOENT; + goto done; + } + + /* + * if the entry is 2nd and on up + */ + if ((req == RTM_DELETE) && !rt_mpath_deldup(rto, rt)) + panic ("rtrequest1: rt_mpath_deldup"); + RT_LOCK(rt); + RT_ADDREF(rt); + if (req == RTM_DELETE) { + rt->rt_flags &= ~RTF_UP; + /* + * One more rtentry floating around that is not + * linked to the routing table. rttrash will be decremented + * when RTFREE(rt) is eventually called. + */ + V_rttrash++; + + } + +nondelete: + if (req != RTM_DELETE) + panic("unrecognized request %d", req); + + + /* + * If the caller wants it, then it can have it, + * but it's up to it to free the rtentry as we won't be + * doing it. + */ + if (ret_nrt) { + *ret_nrt = rt; + RT_UNLOCK(rt); + } else + RTFREE_LOCKED(rt); +done: + return (error); +} +#endif + int rtrequest1_fib(int req, struct rt_addrinfo *info, struct rtentry **ret_nrt, u_int fibnum) @@ -841,65 +938,15 @@ rtrequest1_fib(int req, struct rt_addrin switch (req) { case RTM_DELETE: #ifdef RADIX_MPATH - /* - * if we got multipath routes, we require users to specify - * a matching RTAX_GATEWAY. - */ if (rn_mpath_capable(rnh)) { - struct rtentry *rto = NULL; - - rn = rnh->rnh_matchaddr(dst, rnh); - if (rn == NULL) - senderr(ESRCH); - rto = rt = RNTORT(rn); - rt = rt_mpath_matchgate(rt, gateway); - if (!rt) - senderr(ESRCH); - /* - * this is the first entry in the chain - */ - if (rto == rt) { - rn = rn_mpath_next((struct radix_node *)rt); - /* - * there is another entry, now it's active - */ - if (rn) { - rto = RNTORT(rn); - RT_LOCK(rto); - rto->rt_flags |= RTF_UP; - RT_UNLOCK(rto); - } else if (rt->rt_flags & RTF_GATEWAY) { - /* - * For gateway routes, we need to - * make sure that we we are deleting - * the correct gateway. - * rt_mpath_matchgate() does not - * check the case when there is only - * one route in the chain. - */ - if (gateway && - (rt->rt_gateway->sa_len != gateway->sa_len || - memcmp(rt->rt_gateway, gateway, gateway->sa_len))) - senderr(ESRCH); - } - /* - * use the normal delete code to remove - * the first entry - */ - goto normal_rtdel; - } + error = rn_mpath_update(req, info, rnh, ret_nrt); /* - * if the entry is 2nd and on up + * "bad" holds true for the success case + * as well */ - if (!rt_mpath_deldup(rto, rt)) - panic ("rtrequest1: rt_mpath_deldup"); - RT_LOCK(rt); - RT_ADDREF(rt); - rt->rt_flags &= ~RTF_UP; - goto deldone; /* done with the RTM_DELETE command */ + if (error != ENOENT) + goto bad; } - -normal_rtdel: #endif /* * Remove the item from the tree and return it. @@ -921,9 +968,6 @@ normal_rtdel: if ((ifa = rt->rt_ifa) && ifa->ifa_rtrequest) ifa->ifa_rtrequest(RTM_DELETE, rt, info); -#ifdef RADIX_MPATH -deldone: -#endif /* * One more rtentry floating around that is not * linked to the routing table. rttrash will be decremented @@ -951,11 +995,13 @@ deldone: case RTM_ADD: if ((flags & RTF_GATEWAY) && !gateway) senderr(EINVAL); - if (dst && gateway && (dst->sa_family != gateway->sa_family) && - (gateway->sa_family != AF_UNSPEC) && (gateway->sa_family != AF_LINK)) + if (dst && gateway && (dst->sa_family != gateway->sa_family) + && (gateway->sa_family != AF_UNSPEC) + && (gateway->sa_family != AF_LINK)) senderr(EINVAL); - if (info->rti_ifa == NULL && (error = rt_getifa_fib(info, fibnum))) + if (info->rti_ifa == NULL && + (error = rt_getifa_fib(info, fibnum))) senderr(error); ifa = info->rti_ifa; rt = uma_zalloc(rtzone, M_NOWAIT | M_ZERO); @@ -996,6 +1042,7 @@ deldone: IFAREF(ifa); rt->rt_ifa = ifa; rt->rt_ifp = ifa->ifa_ifp; + rt->rt_rmx.rmx_weight = 1; #ifdef RADIX_MPATH /* do not permit exactly the same dst/mask/gw pair */ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/route.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 02:38:38 2009 (r189342) @@ -59,6 +59,7 @@ struct rt_metrics_lite { u_long rmx_mtu; /* MTU for this path */ u_long rmx_expire; /* lifetime for route, e.g. redirect */ u_long rmx_pksent; /* packets sent using this route */ + u_long rmx_weight; /* absolute weight */ }; struct rt_metrics { @@ -72,7 +73,8 @@ struct rt_metrics { u_long rmx_rtt; /* estimated round trip time */ u_long rmx_rttvar; /* estimated rtt variance */ u_long rmx_pksent; /* packets sent using this route */ - u_long rmx_filler[4]; /* will be used for T/TCP later */ + u_long rmx_weight; /* route weight */ + u_long rmx_filler[3]; /* will be used for T/TCP later */ }; /* @@ -194,13 +196,15 @@ struct ortentry { #define RTF_LOCAL 0x200000 /* route represents a local address */ #define RTF_BROADCAST 0x400000 /* route represents a bcast address */ #define RTF_MULTICAST 0x800000 /* route represents a mcast address */ - /* 0x1000000 and up unassigned */ -#define RTF_RNH_LOCKED 0x40000000 /* radix node head locked by caller */ + /* 0x8000000 and up unassigned */ +#define RTF_STICKY 0x10000000 /* always route dst->src */ + +#define RTF_RNH_LOCKED 0x40000000 /* radix node head is locked */ /* Mask of RTF flags that are allowed to be modified by RTM_CHANGE. */ #define RTF_FMASK \ (RTF_PROTO1 | RTF_PROTO2 | RTF_PROTO3 | RTF_BLACKHOLE | \ - RTF_REJECT | RTF_STATIC) + RTF_REJECT | RTF_STATIC | RTF_STICKY) /* * Routing statistics. @@ -226,12 +230,11 @@ struct rt_msghdr { int rtm_seq; /* for sender to identify action */ int rtm_errno; /* why failed */ int rtm_fmask; /* bitmask used in RTM_CHANGE message */ -#define rtm_use rtm_fmask /* deprecated, use rtm_rmx->rmx_pksent */ u_long rtm_inits; /* which metrics we are initializing */ struct rt_metrics rtm_rmx; /* metrics themselves */ }; -#define RTM_VERSION 5 /* Up the ante and ignore older versions */ +#define RTM_VERSION 6 /* Up the ante and ignore older versions */ /* * Message types. @@ -266,6 +269,7 @@ struct rt_msghdr { #define RTV_SSTHRESH 0x20 /* init or lock _ssthresh */ #define RTV_RTT 0x40 /* init or lock _rtt */ #define RTV_RTTVAR 0x80 /* init or lock _rttvar */ +#define RTV_WEIGHT 0x100 /* init or lock _weight */ /* * Bitmask values for rtm_addrs. @@ -273,7 +277,7 @@ struct rt_msghdr { #define RTA_DST 0x1 /* destination sockaddr present */ #define RTA_GATEWAY 0x2 /* gateway sockaddr present */ #define RTA_NETMASK 0x4 /* netmask sockaddr present */ -#define RTA_GENMASK 0x8 /* cloning mask sockaddr present */ +#define RTA_SPARE 0x8 /* unused */ #define RTA_IFP 0x10 /* interface name sockaddr present */ #define RTA_IFA 0x20 /* interface addr sockaddr present */ #define RTA_AUTHOR 0x40 /* sockaddr for author of redirect */ @@ -285,7 +289,7 @@ struct rt_msghdr { #define RTAX_DST 0 /* destination sockaddr present */ #define RTAX_GATEWAY 1 /* gateway sockaddr present */ #define RTAX_NETMASK 2 /* netmask sockaddr present */ -#define RTAX_GENMASK 3 /* cloning mask sockaddr present */ +#define RTAX_SPARE 3 /* spare field */ #define RTAX_IFP 4 /* interface name sockaddr present */ #define RTAX_IFA 5 /* interface addr sockaddr present */ #define RTAX_AUTHOR 6 /* sockaddr for author of redirect */ @@ -293,11 +297,11 @@ struct rt_msghdr { #define RTAX_MAX 8 /* size of array to allocate */ struct rt_addrinfo { - int rti_addrs; - struct sockaddr *rti_info[RTAX_MAX]; - int rti_flags; - struct ifaddr *rti_ifa; - struct ifnet *rti_ifp; + int rti_addrs; + struct sockaddr *rti_info[RTAX_MAX]; + int rti_flags; + struct ifaddr *rti_ifa; + struct ifnet *rti_ifp; }; /* Modified: user/kmacy/HEAD_fast_net_merge/sys/net/rtsock.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/rtsock.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sys/net/rtsock.c Wed Mar 4 02:38:38 2009 (r189342) @@ -601,7 +601,6 @@ route_output(struct mbuf *m, struct sock info.rti_info[RTAX_DST] = rt_key(rt); info.rti_info[RTAX_GATEWAY] = rt->rt_gateway; info.rti_info[RTAX_NETMASK] = rt_mask(rt); - info.rti_info[RTAX_GENMASK] = 0; if (rtm->rtm_addrs & (RTA_IFP | RTA_IFA)) { ifp = rt->rt_ifp; if (ifp) { @@ -637,7 +636,6 @@ route_output(struct mbuf *m, struct sock } (void)rt_msg2(rtm->rtm_type, &info, (caddr_t)rtm, NULL); rtm->rtm_flags = rt->rt_flags; - rtm->rtm_use = 0; rt_getmetrics(&rt->rt_rmx, &rtm->rtm_rmx); rtm->rtm_addrs = info.rti_addrs; break; @@ -691,10 +689,8 @@ route_output(struct mbuf *m, struct sock rt->rt_ifp = info.rti_ifp; } /* Allow some flags to be toggled on change. */ - if (rtm->rtm_fmask & RTF_FMASK) - rt->rt_flags = (rt->rt_flags & - ~rtm->rtm_fmask) | - (rtm->rtm_flags & rtm->rtm_fmask); + rt->rt_flags = (rt->rt_flags & ~RTF_FMASK) | + (rtm->rtm_flags & RTF_FMASK); rt_setmetrics(rtm->rtm_inits, &rtm->rtm_rmx, &rt->rt_rmx); rtm->rtm_index = rt->rt_ifp->if_index; @@ -767,12 +763,14 @@ static void rt_setmetrics(u_long which, const struct rt_metrics *in, struct rt_metrics_lite *out) { -#define metric(f, e) if (which & (f)) out->e = in->e; +#define metric(f, e) if (which & (f)) { printf("setting 0x%x", f); out->e = in->e; } + /* * Only these are stored in the routing entry since introduction * of tcp hostcache. The rest is ignored. */ metric(RTV_MTU, rmx_mtu); + metric(RTV_WEIGHT, rmx_weight); /* Userland -> kernel timebase conversion. */ if (which & RTV_EXPIRE) out->rmx_expire = in->rmx_expire ? @@ -786,6 +784,7 @@ rt_getmetrics(const struct rt_metrics_li #define metric(e) out->e = in->e; bzero(out, sizeof(*out)); metric(rmx_mtu); + metric(rmx_weight); /* Kernel -> userland timebase conversion. */ out->rmx_expire = in->rmx_expire ? in->rmx_expire - time_uptime + time_second : 0; @@ -1245,7 +1244,6 @@ sysctl_dumpentry(struct radix_node *rn, info.rti_info[RTAX_DST] = rt_key(rt); info.rti_info[RTAX_GATEWAY] = rt->rt_gateway; info.rti_info[RTAX_NETMASK] = rt_mask(rt); - info.rti_info[RTAX_GENMASK] = 0; if (rt->rt_ifp) { info.rti_info[RTAX_IFP] = rt->rt_ifp->if_addr->ifa_addr; info.rti_info[RTAX_IFA] = rt->rt_ifa->ifa_addr; @@ -1257,7 +1255,10 @@ sysctl_dumpentry(struct radix_node *rn, struct rt_msghdr *rtm = (struct rt_msghdr *)w->w_tmem; rtm->rtm_flags = rt->rt_flags; - rtm->rtm_use = rt->rt_rmx.rmx_pksent; + /* + * let's be honest about this being a retarded hack + */ + rtm->rtm_fmask = rt->rt_rmx.rmx_pksent; rt_getmetrics(&rt->rt_rmx, &rtm->rtm_rmx); rtm->rtm_index = rt->rt_ifp->if_index; rtm->rtm_errno = rtm->rtm_pid = rtm->rtm_seq = 0; Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_input.c Wed Mar 4 02:38:38 2009 (r189342) @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include "opt_route.h" #include "opt_mac.h" #include "opt_carp.h" +#include "opt_mpath.h" #include #include @@ -340,7 +341,7 @@ ip_init(void) netisr_register(NETISR_IP, ip_input, &ipintrq, 0); ipv4_ft = flowtable_alloc(ip_pcpu_flowtable_size, FL_PCPU); - ipv4_forward_ft = flowtable_alloc(ip_global_flowtable_size, FL_HASH_PORTS); + ipv4_forward_ft = flowtable_alloc(ip_global_flowtable_size, FL_HASH_PORTS|FL_PCPU); } void Modified: user/kmacy/HEAD_fast_net_merge/usr.sbin/route6d/route6d.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/usr.sbin/route6d/route6d.c Wed Mar 4 02:12:29 2009 (r189341) +++ user/kmacy/HEAD_fast_net_merge/usr.sbin/route6d/route6d.c Wed Mar 4 02:38:38 2009 (r189342) @@ -2661,10 +2661,6 @@ rt_entry(rtm, again) sin6_mask = (struct sockaddr_in6 *)rtmp; rtmp += ROUNDUP(sin6_mask->sin6_len); } - if (rtm->rtm_addrs & RTA_GENMASK) { - sin6_genmask = (struct sockaddr_in6 *)rtmp; - rtmp += ROUNDUP(sin6_genmask->sin6_len); - } if (rtm->rtm_addrs & RTA_IFP) { sin6_ifp = (struct sockaddr_in6 *)rtmp; rtmp += ROUNDUP(sin6_ifp->sin6_len); From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 20:58:24 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id B3C56106564A; Wed, 4 Mar 2009 20:58:24 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 9E0D18FC12; Wed, 4 Mar 2009 20:58:24 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n24KwOKs053011; Wed, 4 Mar 2009 20:58:24 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n24KwMbT052971; Wed, 4 Mar 2009 20:58:22 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903042058.n24KwMbT052971@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 20:58:22 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189372 - in user/kmacy/HEAD_fast_net_merge: contrib/ipfilter/tools sys/contrib/ipfilter/netinet sys/contrib/pf/net sys/dev/iicbus sys/dev/lmc sys/dev/ppbus sys/net sys/net80211 sys/net... X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 20:58:25 -0000 Author: kmacy Date: Wed Mar 4 20:58:22 2009 New Revision: 189372 URL: http://svn.freebsd.org/changeset/base/189372 Log: Change if_output interface to allow passing cached lle down to L2: 186574, 186577, 186673 186574: - Change if_output interface to allow passing cached lle down to L2 186577: - used cache lle where possible 186673: - fix if_output calls in modules Modified: user/kmacy/HEAD_fast_net_merge/contrib/ipfilter/tools/ipftest.c user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_compat.h user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_sync.c user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pflog.c user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pfsync.c user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/pf.c user/kmacy/HEAD_fast_net_merge/sys/dev/iicbus/if_ic.c user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.c user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.h user/kmacy/HEAD_fast_net_merge/sys/dev/ppbus/if_plip.c user/kmacy/HEAD_fast_net_merge/sys/net/bpf.c user/kmacy/HEAD_fast_net_merge/sys/net/ethernet.h user/kmacy/HEAD_fast_net_merge/sys/net/if_arc.h user/kmacy/HEAD_fast_net_merge/sys/net/if_arcsubr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_atm.h user/kmacy/HEAD_fast_net_merge/sys/net/if_atmsubr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_disc.c user/kmacy/HEAD_fast_net_merge/sys/net/if_enc.c user/kmacy/HEAD_fast_net_merge/sys/net/if_ethersubr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_faith.c user/kmacy/HEAD_fast_net_merge/sys/net/if_fddisubr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_fwsubr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.c user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.h user/kmacy/HEAD_fast_net_merge/sys/net/if_gre.c user/kmacy/HEAD_fast_net_merge/sys/net/if_iso88025subr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.c user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.h user/kmacy/HEAD_fast_net_merge/sys/net/if_loop.c user/kmacy/HEAD_fast_net_merge/sys/net/if_spppsubr.c user/kmacy/HEAD_fast_net_merge/sys/net/if_stf.c user/kmacy/HEAD_fast_net_merge/sys/net/if_tun.c user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h user/kmacy/HEAD_fast_net_merge/sys/net/iso88025.h user/kmacy/HEAD_fast_net_merge/sys/net/route.h user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211.c user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211_output.c user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211_proto.h user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211_var.h user/kmacy/HEAD_fast_net_merge/sys/netatalk/aarp.c user/kmacy/HEAD_fast_net_merge/sys/netatalk/ddp_output.c user/kmacy/HEAD_fast_net_merge/sys/netgraph/atm/ng_atm.c user/kmacy/HEAD_fast_net_merge/sys/netgraph/ng_fec.c user/kmacy/HEAD_fast_net_merge/sys/netgraph/ng_gif.c user/kmacy/HEAD_fast_net_merge/sys/netgraph/ng_iface.c user/kmacy/HEAD_fast_net_merge/sys/netinet/if_ether.c user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_carp.c user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_fastfwd.c user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c user/kmacy/HEAD_fast_net_merge/sys/netinet6/in6.h user/kmacy/HEAD_fast_net_merge/sys/netinet6/ip6_mroute.c user/kmacy/HEAD_fast_net_merge/sys/netinet6/nd6.c user/kmacy/HEAD_fast_net_merge/sys/netipx/ipx_outputfl.c user/kmacy/HEAD_fast_net_merge/sys/netnatm/natm.c Modified: user/kmacy/HEAD_fast_net_merge/contrib/ipfilter/tools/ipftest.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/contrib/ipfilter/tools/ipftest.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/contrib/ipfilter/tools/ipftest.c Wed Mar 4 20:58:22 2009 (r189372) @@ -9,6 +9,9 @@ #include "ipt.h" #include #include +#include +#include +#include #if !defined(lint) static const char sccsid[] = "@(#)ipt.c 1.19 6/3/96 (C) 1993-2000 Darren Reed"; @@ -282,16 +285,20 @@ char *argv[]; printf("--------------"); } else if ((opts & (OPT_BRIEF|OPT_NAT)) == (OPT_NAT|OPT_BRIEF)) printpacket(ip); - if (dir && (ifp != NULL) && IP_V(ip) && (m != NULL)) + if (dir && (ifp != NULL) && IP_V(ip) && (m != NULL)) { + struct route ro; + bzero(&ro, sizeof(ro)); #if defined(__sgi) && (IRIX < 60500) (*ifp->if_output)(ifp, (void *)m, NULL); #else # if TRU64 >= 1885 (*ifp->if_output)(ifp, (void *)m, NULL, 0, 0); # else - (*ifp->if_output)(ifp, (void *)m, NULL, 0); + (*ifp->if_output)(ifp, (void *)m, &ro); # endif #endif + } + if ((opts & (OPT_BRIEF|OPT_NAT)) != (OPT_NAT|OPT_BRIEF)) putchar('\n'); dir = 0; Modified: user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_compat.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_compat.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_compat.h Wed Mar 4 20:58:22 2009 (r189372) @@ -1825,7 +1825,7 @@ typedef struct tcpiphdr tcpiphdr_t; #endif #ifndef offsetof -# define offsetof(t,m) (int)((&((t *)0L)->m)) +# define offsetof(t,m) (size_t)((&((t *)0L)->m)) #endif /* Modified: user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c Wed Mar 4 20:58:22 2009 (r189372) @@ -1045,8 +1045,8 @@ frdest_t *fdp; if (!ip->ip_sum) ip->ip_sum = in_cksum(m, hlen); - error = (*ifp->if_output)(ifp, m, (struct sockaddr *)dst, - ro->ro_rt); + bcopy(dst, &ro->ro_dst, sizeof(dst)); + error = (*ifp->if_output)(ifp, m, ro); goto done; } /* @@ -1122,12 +1122,12 @@ frdest_t *fdp; ip->ip_sum = 0; ip->ip_sum = in_cksum(m0, hlen); sendorfree: + bcopy(dst, &ro->ro_dst, sizeof(dst)); for (m = m0; m; m = m0) { m0 = m->m_act; m->m_act = 0; if (error == 0) - error = (*ifp->if_output)(ifp, m, - (struct sockaddr *)dst, ro->ro_rt); + error = (*ifp->if_output)(ifp, m, ro); else FREE_MB_T(m); } Modified: user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_sync.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_sync.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/contrib/ipfilter/netinet/ip_sync.c Wed Mar 4 20:58:22 2009 (r189372) @@ -729,9 +729,9 @@ void *data; } nat = (nat_t *)data; - bzero((char *)n, offsetof(nat_t, nat_age)); + bzero((char *)n, __offsetof(nat_t, nat_age)); bcopy((char *)&nat->nat_age, (char *)&n->nat_age, - sizeof(*n) - offsetof(nat_t, nat_age)); + sizeof(*n) - __offsetof(nat_t, nat_age)); ipfsync_natorder(0, n); n->nat_sync = sl; Modified: user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pflog.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pflog.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pflog.c Wed Mar 4 20:58:22 2009 (r189372) @@ -112,8 +112,7 @@ __FBSDID("$FreeBSD$"); #endif void pflogattach(int); -int pflogoutput(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +int pflogoutput(struct ifnet *, struct mbuf *, struct route *); int pflogioctl(struct ifnet *, u_long, caddr_t); void pflogstart(struct ifnet *); #ifdef __FreeBSD__ @@ -286,8 +285,7 @@ pflogstart(struct ifnet *ifp) } int -pflogoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +pflogoutput(struct ifnet *ifp, struct mbuf *m, struct route *ro) { m_freem(m); return (0); Modified: user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pfsync.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pfsync.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/if_pfsync.c Wed Mar 4 20:58:22 2009 (r189372) @@ -151,8 +151,7 @@ int pfsync_insert_net_state(struct pfsyn #ifdef PFSYNC_TDB void pfsync_update_net_tdb(struct pfsync_tdb *); #endif -int pfsyncoutput(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +int pfsyncoutput(struct ifnet *, struct mbuf *, struct route *); int pfsyncioctl(struct ifnet *, u_long, caddr_t); void pfsyncstart(struct ifnet *); @@ -1082,8 +1081,7 @@ done: } int -pfsyncoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +pfsyncoutput(struct ifnet *ifp, struct mbuf *m, struct route *ro) { m_freem(m); return (0); Modified: user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/pf.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/pf.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/contrib/pf/net/pf.c Wed Mar 4 20:58:22 2009 (r189372) @@ -6244,8 +6244,9 @@ pf_route(struct mbuf **m, struct pf_rule ip->ip_sum = in_cksum(m0, ip->ip_hl << 2); } } + bcopy(dst, &ro->ro_dst, sizeof(*dst)); PF_UNLOCK(); - error = (*ifp->if_output)(ifp, m0, sintosa(dst), ro->ro_rt); + error = (*ifp->if_output)(ifp, m0, ro); PF_LOCK(); goto done; } @@ -6281,6 +6282,8 @@ pf_route(struct mbuf **m, struct pf_rule } if (ntohs(ip->ip_len) <= ifp->if_mtu) { + struct route ro; + if ((ifp->if_capabilities & IFCAP_CSUM_IPv4) && ifp->if_bridge == NULL) { m0->m_pkthdr.csum_flags |= M_IPV4_CSUM_OUT; @@ -6294,7 +6297,9 @@ pf_route(struct mbuf **m, struct pf_rule V_tcpstat.tcps_outhwcsum++; else if (m0->m_pkthdr.csum_flags & M_UDPV4_CSUM_OUT) V_udpstat.udps_outhwcsum++; - error = (*ifp->if_output)(ifp, m0, sintosa(dst), NULL); + + bcopy(dst, &ro->ro_dst, sizeof(*dst)); + error = (*ifp->if_output)(ifp, m0, ro); goto done; } #endif @@ -6345,9 +6350,11 @@ pf_route(struct mbuf **m, struct pf_rule m0->m_nextpkt = 0; #ifdef __FreeBSD__ if (error == 0) { + bcopy(dst, &ro->ro_dst, sizeof(*dst)); + ro->ro_lle = NULL; + ro->ro_rt = NULL; PF_UNLOCK(); - error = (*ifp->if_output)(ifp, m0, sintosa(dst), - NULL); + error = (*ifp->if_output)(ifp, m0, ro); PF_LOCK(); } else #else Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/iicbus/if_ic.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/iicbus/if_ic.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/iicbus/if_ic.c Wed Mar 4 20:58:22 2009 (r189372) @@ -102,9 +102,7 @@ static int icprobe(device_t); static int icattach(device_t); static int icioctl(struct ifnet *, u_long, caddr_t); -static int icoutput(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); - +static int icoutput(struct ifnet *, struct mbuf *, struct route *); static int icintr(device_t, int, char *); static device_method_t ic_methods[] = { @@ -353,8 +351,7 @@ icintr(device_t dev, int event, char *pt * icoutput() */ static int -icoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +icoutput(struct ifnet *ifp, struct mbuf *m, struct route *ro) { struct ic_softc *sc = ifp->if_softc; device_t icdev = sc->ic_dev; @@ -363,6 +360,7 @@ icoutput(struct ifnet *ifp, struct mbuf struct mbuf *mm; u_char *cp; u_int32_t hdr; + struct sockaddr *dst = &ro->ro_dst; /* BPF writes need to be handled specially. */ if (dst->sa_family == AF_UNSPEC) Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.c Wed Mar 4 20:58:22 2009 (r189372) @@ -114,6 +114,7 @@ # include # include # include +# include # include # include # include @@ -4582,8 +4583,7 @@ lmc_ifnet_start(struct ifnet *ifp) /* RAWIP mode is the only time this is used. */ /* Called from a syscall (user context; no spinlocks). */ static int -lmc_raw_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt) +lmc_raw_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { softc_t *sc = IFP2SC(ifp); int error = 0; Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/lmc/if_lmc.h Wed Mar 4 20:58:22 2009 (r189372) @@ -1594,8 +1594,7 @@ static void core_watchdog(softc_t *); static int lmc_raw_ioctl(struct ifnet *, u_long, caddr_t); static int lmc_ifnet_ioctl(struct ifnet *, u_long, caddr_t); static void lmc_ifnet_start(struct ifnet *); -static int lmc_raw_output(struct ifnet *, struct mbuf *, - struct sockaddr *, struct rtentry *); +static int lmc_raw_output(struct ifnet *, struct mbuf *, struct route *ro); static void lmc_ifnet_watchdog(struct ifnet *); # ifdef __OpenBSD__ static int ifmedia_change(struct ifnet *); Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/ppbus/if_plip.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/ppbus/if_plip.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/ppbus/if_plip.c Wed Mar 4 20:58:22 2009 (r189372) @@ -99,6 +99,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -172,8 +173,7 @@ static u_char *ctxmith; /* Functions for the lp# interface */ static int lpinittables(void); static int lpioctl(struct ifnet *, u_long, caddr_t); -static int lpoutput(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +static int lpoutput(struct ifnet *, struct mbuf *, struct route *); static void lpstop(struct lp_data *); static void lp_intr(void *); static int lp_module_handler(module_t, int, void *); @@ -677,8 +677,7 @@ lpoutbyte(u_char byte, int spin, device_ } static int -lpoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +lpoutput(struct ifnet *ifp, struct mbuf *m, struct route *ro) { struct lp_data *sc = ifp->if_softc; device_t dev = sc->sc_dev; Modified: user/kmacy/HEAD_fast_net_merge/sys/net/bpf.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/bpf.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/bpf.c Wed Mar 4 20:58:22 2009 (r189372) @@ -73,6 +73,7 @@ __FBSDID("$FreeBSD$"); #endif #include #include +#include #include #include @@ -822,7 +823,8 @@ bpfwrite(struct cdev *dev, struct uio *u struct mbuf *m, *mc; struct sockaddr dst; int error, hlen; - + struct route ro; + error = devfs_get_cdevpriv((void **)&d); if (error != 0) return (error); @@ -883,7 +885,10 @@ bpfwrite(struct cdev *dev, struct uio *u BPFD_UNLOCK(d); #endif - error = (*ifp->if_output)(ifp, m, &dst, NULL); + bcopy(&dst, &ro.ro_dst, sizeof(dst)); + ro.ro_rt = NULL; + ro.ro_lle = NULL; + error = (*ifp->if_output)(ifp, m, &ro); if (error) d->bd_wdcount++; Modified: user/kmacy/HEAD_fast_net_merge/sys/net/ethernet.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/ethernet.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/ethernet.h Wed Mar 4 20:58:22 2009 (r189372) @@ -368,6 +368,7 @@ struct mbuf; struct rtentry; struct sockaddr; struct bpf_if; +struct route; extern uint32_t ether_crc32_le(const uint8_t *, size_t); extern uint32_t ether_crc32_be(const uint8_t *, size_t); @@ -375,8 +376,7 @@ extern void ether_demux(struct ifnet *, extern void ether_ifattach(struct ifnet *, const u_int8_t *); extern void ether_ifdetach(struct ifnet *); extern int ether_ioctl(struct ifnet *, u_long, caddr_t); -extern int ether_output(struct ifnet *, - struct mbuf *, struct sockaddr *, struct rtentry *); +extern int ether_output(struct ifnet *, struct mbuf *, struct route *); extern int ether_output_frame(struct ifnet *, struct mbuf *); extern char *ether_sprintf(const u_int8_t *); void ether_vlan_mtap(struct bpf_if *, struct mbuf *, Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_arc.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_arc.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_arc.h Wed Mar 4 20:58:22 2009 (r189372) @@ -132,8 +132,7 @@ void arc_ifdetach(struct ifnet *); void arc_storelladdr(struct ifnet *, u_int8_t); int arc_isphds(u_int8_t); void arc_input(struct ifnet *, struct mbuf *); -int arc_output(struct ifnet *, struct mbuf *, - struct sockaddr *, struct rtentry *); +int arc_output(struct ifnet *, struct mbuf *, struct route *); int arc_ioctl(struct ifnet *, int, caddr_t); void arc_frag_init(struct ifnet *); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_arcsubr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_arcsubr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_arcsubr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -101,8 +101,7 @@ u_int8_t arcbroadcastaddr = 0; * Assumes that ifp is actually pointer to arccom structure. */ int -arc_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt0) +arc_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { struct arc_header *ah; int error; @@ -110,7 +109,9 @@ arc_output(struct ifnet *ifp, struct mbu int loop_copy = 0; int isphds; struct llentry *lle; - + struct sockaddr *dst = &ro->ro_dst; + struct rtentry *rt0 = ro->ro_rt; + if (!((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING))) return(ENETDOWN); /* m, m1 aren't initialized yet */ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_atm.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_atm.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_atm.h Wed Mar 4 20:58:22 2009 (r189372) @@ -292,8 +292,7 @@ void atm_ifattach(struct ifnet *); void atm_ifdetach(struct ifnet *); void atm_input(struct ifnet *, struct atm_pseudohdr *, struct mbuf *, void *); -int atm_output(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +int atm_output(struct ifnet *, struct mbuf *, struct route *); struct atmio_vcctable *atm_getvccs(struct atmio_vcc **, u_int, u_int, struct mtx *, int); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_atmsubr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_atmsubr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_atmsubr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -122,8 +122,7 @@ MALLOC_DEFINE(M_IFATM, "ifatm", "atm int * rt0 must also be NULL. */ int -atm_output(struct ifnet *ifp, struct mbuf *m0, struct sockaddr *dst, - struct rtentry *rt0) +atm_output(struct ifnet *ifp, struct mbuf *m0, struct route *ro) { u_int16_t etype = 0; /* if using LLC/SNAP */ int error = 0, sz; @@ -132,6 +131,8 @@ atm_output(struct ifnet *ifp, struct mbu struct atmllc *atmllc; struct atmllc *llc_hdr = NULL; u_int32_t atm_flags; + struct sockaddr *dst = &ro->ro_dst; + struct rtentry *rt0 = ro->ro_rt; #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_disc.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_disc.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_disc.c Wed Mar 4 20:58:22 2009 (r189372) @@ -65,8 +65,7 @@ struct disc_softc { struct ifnet *sc_ifp; }; -static int discoutput(struct ifnet *, struct mbuf *, - struct sockaddr *, struct rtentry *); +static int discoutput(struct ifnet *, struct mbuf *, struct route *); static void discrtrequest(int, struct rtentry *, struct rt_addrinfo *); static int discioctl(struct ifnet *, u_long, caddr_t); static int disc_clone_create(struct if_clone *, int, caddr_t); @@ -155,13 +154,13 @@ static moduledata_t disc_mod = { DECLARE_MODULE(if_disc, disc_mod, SI_SUB_PSEUDO, SI_ORDER_ANY); static int -discoutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +discoutput(struct ifnet *ifp, struct mbuf *m, struct route *ro) { u_int32_t af; + struct sockaddr *dst = &ro->ro_dst; + M_ASSERTPKTHDR(m); - /* BPF writes need to be handled specially. */ if (dst->sa_family == AF_UNSPEC) { bcopy(dst->sa_data, &af, sizeof(af)); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_enc.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_enc.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_enc.c Wed Mar 4 20:58:22 2009 (r189372) @@ -84,8 +84,7 @@ struct enc_softc { }; static int enc_ioctl(struct ifnet *, u_long, caddr_t); -static int enc_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt); +static int enc_output(struct ifnet *ifp, struct mbuf *m, struct route *ro); static int enc_clone_create(struct if_clone *, int, caddr_t); static void enc_clone_destroy(struct ifnet *); @@ -184,8 +183,7 @@ static moduledata_t enc_mod = { DECLARE_MODULE(enc, enc_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY); static int -enc_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +enc_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { m_freem(m); return (0); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_ethersubr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_ethersubr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_ethersubr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -162,18 +162,20 @@ static int ether_ipfw; * packet leaves a multiple of 512 bytes of data in remainder. */ int -ether_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt0) +ether_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { short type; - int error, hdrcmplt = 0; + int error = 0, hdrcmplt = 0; u_char esrc[ETHER_ADDR_LEN], edst[ETHER_ADDR_LEN]; - struct llentry *lle = NULL; + struct llentry *lle = ro->ro_lle; struct ether_header *eh; struct pf_mtag *t; int loop_copy = 1; + struct sockaddr *dst = &ro->ro_dst; + struct rtentry *rt0 = ro->ro_rt; int hlen; /* link layer header length */ + #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); if (error) @@ -191,7 +193,10 @@ ether_output(struct ifnet *ifp, struct m switch (dst->sa_family) { #ifdef INET case AF_INET: - error = arpresolve(ifp, rt0, m, dst, edst, &lle); + if (lle != NULL && (lle->la_flags & LLE_VALID)) + memcpy(edst, &lle->ll_addr.mac16, ETHER_ADDR_LEN); + else + error = arpresolve(ifp, rt0, m, dst, edst, &lle); if (error) return (error == EWOULDBLOCK ? 0 : error); type = htons(ETHERTYPE_IP); @@ -226,7 +231,10 @@ ether_output(struct ifnet *ifp, struct m #endif #ifdef INET6 case AF_INET6: - error = nd6_storelladdr(ifp, m, dst, (u_char *)edst, &lle); + if (lle != NULL && (lle->la_flags & LLE_VALID)) + memcpy(edst, &lle->ll_addr.mac16, ETHER_ADDR_LEN); + else + error = nd6_storelladdr(ifp, m, dst, (u_char *)edst, &lle); if (error) return error; type = htons(ETHERTYPE_IPV6); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_faith.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_faith.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_faith.c Wed Mar 4 20:58:22 2009 (r189372) @@ -87,8 +87,7 @@ struct faith_softc { }; static int faithioctl(struct ifnet *, u_long, caddr_t); -int faithoutput(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +int faithoutput(struct ifnet *, struct mbuf *, struct route *); static void faithrtrequest(int, struct rtentry *, struct rt_addrinfo *); #ifdef INET6 static int faithprefix(struct in6_addr *); @@ -188,14 +187,15 @@ faith_clone_destroy(ifp) } int -faithoutput(ifp, m, dst, rt) +faithoutput(ifp, m, ro) struct ifnet *ifp; struct mbuf *m; - struct sockaddr *dst; - struct rtentry *rt; + struct route *ro; { int isr; u_int32_t af; + struct sockaddr *dst = &ro->ro_dst; + struct rtentry *rt = ro->ro_rt; M_ASSERTPKTHDR(m); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_fddisubr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_fddisubr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_fddisubr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -97,8 +97,7 @@ static const u_char fddibroadcastaddr[FD static int fddi_resolvemulti(struct ifnet *, struct sockaddr **, struct sockaddr *); -static int fddi_output(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +static int fddi_output(struct ifnet *, struct mbuf *, struct route *); static void fddi_input(struct ifnet *ifp, struct mbuf *m); #define senderr(e) do { error = (e); goto bad; } while (0) @@ -111,17 +110,18 @@ static void fddi_input(struct ifnet *ifp * Assumes that ifp is actually pointer to arpcom structure. */ static int -fddi_output(ifp, m, dst, rt0) +fddi_output(ifp, m, ro) struct ifnet *ifp; struct mbuf *m; - struct sockaddr *dst; - struct rtentry *rt0; + struct route *ro; { u_int16_t type; int loop_copy = 0, error = 0, hdrcmplt = 0; u_char esrc[FDDI_ADDR_LEN], edst[FDDI_ADDR_LEN]; struct fddi_header *fh; struct llentry *lle; + struct sockaddr *dst = &ro->ro_dst; + struct rtentry *rt0 = ro->ro_rt; #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_fwsubr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_fwsubr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_fwsubr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -76,8 +76,7 @@ struct fw_hwaddr firewire_broadcastaddr }; static int -firewire_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt0) +firewire_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { struct fw_com *fc = IFP2FWC(ifp); int error, type; @@ -89,8 +88,10 @@ firewire_output(struct ifnet *ifp, struc struct mbuf *mtail; int unicast, dgl, foff; static int next_dgl; - struct llentry *lle; - + struct llentry *lle = ro->ro_lle; + struct rtentry *rt0 = ro->ro_rt; + struct sockaddr *dst = &ro->ro_dst; + #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); if (error) Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.c Wed Mar 4 20:58:22 2009 (r189372) @@ -369,17 +369,19 @@ gif_start(struct ifnet *ifp) { struct gif_softc *sc; struct mbuf *m; + struct route ro; sc = ifp->if_softc; - + bcopy(sc->gif_pdst, &ro.ro_dst, sizeof(sc->gif_pdst)); + ro.ro_rt = NULL; + ro.ro_lle = NULL; ifp->if_drv_flags |= IFF_DRV_OACTIVE; for (;;) { IFQ_DEQUEUE(&ifp->if_snd, m); if (m == 0) break; - gif_output(ifp, m, sc->gif_pdst, NULL); - + gif_output(ifp, m, &ro); } ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; @@ -387,11 +389,10 @@ gif_start(struct ifnet *ifp) } int -gif_output(ifp, m, dst, rt) +gif_output(ifp, m, ro) struct ifnet *ifp; struct mbuf *m; - struct sockaddr *dst; - struct rtentry *rt; /* added in net2 */ + struct route *ro; { INIT_VNET_GIF(ifp->if_vnet); struct gif_softc *sc = ifp->if_softc; @@ -399,6 +400,7 @@ gif_output(ifp, m, dst, rt) int error = 0; int gif_called; u_int32_t af; + struct sockaddr *dst = &ro->ro_dst; #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_gif.h Wed Mar 4 20:58:22 2009 (r189372) @@ -103,8 +103,7 @@ struct etherip_header { /* Prototypes */ void gif_input(struct mbuf *, int, struct ifnet *); -int gif_output(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +int gif_output(struct ifnet *, struct mbuf *, struct route *ro); int gif_ioctl(struct ifnet *, u_long, caddr_t); int gif_set_tunnel(struct ifnet *, struct sockaddr *, struct sockaddr *); void gif_delete_tunnel(struct ifnet *); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_gre.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_gre.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_gre.c Wed Mar 4 20:58:22 2009 (r189372) @@ -109,8 +109,7 @@ struct gre_softc_head gre_softc_list; static int gre_clone_create(struct if_clone *, int, caddr_t); static void gre_clone_destroy(struct ifnet *); static int gre_ioctl(struct ifnet *, u_long, caddr_t); -static int gre_output(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *rt); +static int gre_output(struct ifnet *, struct mbuf *, struct route *); IFC_SIMPLE_DECLARE(gre, 0); @@ -239,8 +238,7 @@ gre_clone_destroy(ifp) * given by sc->g_proto. See also RFC 1701 and RFC 2004 */ static int -gre_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +gre_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { #ifdef INET6 INIT_VNET_INET(ifp->if_vnet); @@ -255,6 +253,7 @@ gre_output(struct ifnet *ifp, struct mbu struct mobile_h mob_h; u_int32_t af; int extra = 0; + struct sockaddr *dst = &ro->ro_dst; /* * gre may cause infinite recursion calls when misconfigured. Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_iso88025subr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_iso88025subr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_iso88025subr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -232,11 +232,10 @@ iso88025_ioctl(struct ifnet *ifp, int co * ISO88025 encapsulation */ int -iso88025_output(ifp, m, dst, rt0) +iso88025_output(ifp, m, ro) struct ifnet *ifp; struct mbuf *m; - struct sockaddr *dst; - struct rtentry *rt0; + struct route *ro; { u_int16_t snap_type = 0; int loop_copy = 0, error = 0, rif_len = 0; @@ -245,12 +244,16 @@ iso88025_output(ifp, m, dst, rt0) struct iso88025_header gen_th; struct sockaddr_dl *sdl = NULL; struct llentry *lle; + struct sockaddr *dst; + struct rtentry *rt0; #ifdef MAC error = mac_ifnet_check_transmit(ifp, m); if (error) senderr(error); #endif + rt0 = ro->ro_rt; + dst = &ro->ro_dst; if (ifp->if_flags & IFF_MONITOR) senderr(ENETDOWN); @@ -628,8 +631,9 @@ iso88025_input(ifp, m) case LLC_TEST: case LLC_TEST_P: { - struct sockaddr sa; + struct sockaddr *sa; struct arpcom *ac; + struct route ro; struct iso88025_sockaddr_data *th2; int i; u_char c; @@ -646,8 +650,9 @@ iso88025_input(ifp, m) bcopy((caddr_t)IF_LLADDR(ifp), (caddr_t)th->iso88025_dhost, ISO88025_ADDR_LEN); - sa.sa_family = AF_UNSPEC; - sa.sa_len = sizeof(sa); + sa = &ro.ro_dst; + sa->sa_family = AF_UNSPEC; + sa->sa_len = sizeof(*sa); th2 = (struct iso88025_sockaddr_data *)sa.sa_data; for (i = 0; i < ISO88025_ADDR_LEN; i++) { th2->ether_shost[i] = c = th->iso88025_dhost[i]; @@ -657,7 +662,7 @@ iso88025_input(ifp, m) } th2->ac = TR_AC; th2->fc = TR_LLC_FRAME; - ifp->if_output(ifp, m, &sa, NULL); + ifp->if_output(ifp, m, &ro); return; } default: Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.c Wed Mar 4 20:58:22 2009 (r189372) @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #ifdef INET #include @@ -92,8 +93,7 @@ static struct mbuf *lagg_input(struct if static void lagg_linkstate(struct lagg_softc *); static void lagg_port_state(struct ifnet *, int); static int lagg_port_ioctl(struct ifnet *, u_long, caddr_t); -static int lagg_port_output(struct ifnet *, struct mbuf *, - struct sockaddr *, struct rtentry *); +static int lagg_port_output(struct ifnet *, struct mbuf *, struct route *ro); static void lagg_port_ifdetach(void *arg __unused, struct ifnet *); static int lagg_port_checkstacking(struct lagg_softc *); static void lagg_port2req(struct lagg_port *, struct lagg_reqport *); @@ -675,11 +675,11 @@ fallback: } static int -lagg_port_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt0) +lagg_port_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { struct lagg_port *lp = ifp->if_lagg; struct ether_header *eh; + struct sockaddr *dst = &ro->ro_dst; short type = 0; switch (dst->sa_family) { @@ -696,7 +696,7 @@ lagg_port_output(struct ifnet *ifp, stru */ switch (ntohs(type)) { case ETHERTYPE_PAE: /* EAPOL PAE/802.1x */ - return ((*lp->lp_output)(ifp, m, dst, rt0)); + return ((*lp->lp_output)(ifp, m, ro)); } /* drop any other frames */ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_lagg.h Wed Mar 4 20:58:22 2009 (r189372) @@ -217,8 +217,7 @@ struct lagg_port { /* Redirected callbacks */ int (*lp_ioctl)(struct ifnet *, u_long, caddr_t); - int (*lp_output)(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); + int (*lp_output)(struct ifnet *, struct mbuf *, struct route *); SLIST_ENTRY(lagg_port) lp_entries; }; Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_loop.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_loop.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_loop.c Wed Mar 4 20:58:22 2009 (r189372) @@ -96,8 +96,7 @@ int loioctl(struct ifnet *, u_long, caddr_t); static void lortrequest(int, struct rtentry *, struct rt_addrinfo *); -int looutput(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt); +int looutput(struct ifnet *ifp, struct mbuf *m, struct route *ro); static int lo_clone_create(struct if_clone *, int, caddr_t); static void lo_clone_destroy(struct ifnet *); @@ -176,13 +175,14 @@ static moduledata_t loop_mod = { DECLARE_MODULE(loop, loop_mod, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY); int -looutput(struct ifnet *ifp, struct mbuf *m, struct sockaddr *dst, - struct rtentry *rt) +looutput(struct ifnet *ifp, struct mbuf *m, struct route *ro) { u_int32_t af; #ifdef MAC int error; #endif + struct sockaddr *dst = &ro->ro_dst; + struct rtentry *rt = ro->ro_rt; M_ASSERTPKTHDR(m); /* check if we have the packet header */ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_spppsubr.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_spppsubr.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_spppsubr.c Wed Mar 4 20:58:22 2009 (r189372) @@ -262,9 +262,7 @@ static const u_short interactive_ports[8 struct ifnet *ifp = SP2IFP(sp); \ int debug = ifp->if_flags & IFF_DEBUG -static int sppp_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt); - +static int sppp_output(struct ifnet *ifp, struct mbuf *m, struct route *ro); static void sppp_cisco_send(struct sppp *sp, int type, long par1, long par2); static void sppp_cisco_input(struct sppp *sp, struct mbuf *m); @@ -785,8 +783,7 @@ sppp_ifstart(struct ifnet *ifp) * Enqueue transmit packet. */ static int -sppp_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt) +sppp_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { struct sppp *sp = IFP2SP(ifp); struct ppp_header *h; @@ -796,6 +793,7 @@ sppp_output(struct ifnet *ifp, struct mb int ipproto = PPP_IP; #endif int debug = ifp->if_flags & IFF_DEBUG; + struct sockaddr *dst = &ro->ro_dst; s = splimp(); SPPP_LOCK(sp); Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_stf.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_stf.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_stf.c Wed Mar 4 20:58:22 2009 (r189372) @@ -177,8 +177,7 @@ static char *stfnames[] = {"stf0", "stf" static int stfmodevent(module_t, int, void *); static int stf_encapcheck(const struct mbuf *, int, int, void *); static struct in6_ifaddr *stf_getsrcifa6(struct ifnet *); -static int stf_output(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +static int stf_output(struct ifnet *, struct mbuf *, struct route *); static int isrfc1918addr(struct in_addr *); static int stf_checkaddr4(struct stf_softc *, struct in_addr *, struct ifnet *); @@ -403,11 +402,10 @@ stf_getsrcifa6(ifp) } static int -stf_output(ifp, m, dst, rt) +stf_output(ifp, m, ro) struct ifnet *ifp; struct mbuf *m; - struct sockaddr *dst; - struct rtentry *rt; + struct route *ro; { struct stf_softc *sc; struct sockaddr_in6 *dst6; @@ -415,6 +413,7 @@ stf_output(ifp, m, dst, rt) struct in_addr in4; caddr_t ptr; struct sockaddr_in *dst4; + struct sockaddr *dst; u_int8_t tos; struct ip *ip; struct ip6_hdr *ip6; @@ -431,6 +430,7 @@ stf_output(ifp, m, dst, rt) #endif sc = ifp->if_softc; + dst = &ro->ro_dst; dst6 = (struct sockaddr_in6 *)dst; /* just in case */ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_tun.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_tun.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_tun.c Wed Mar 4 20:58:22 2009 (r189372) @@ -128,8 +128,7 @@ static void tuncreate(const char *name, static int tunifioctl(struct ifnet *, u_long, caddr_t); static int tuninit(struct ifnet *); static int tunmodevent(module_t, int, void *); -static int tunoutput(struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *rt); +static int tunoutput(struct ifnet *, struct mbuf *, struct route *ro); static void tunstart(struct ifnet *); static int tun_clone_create(struct if_clone *, int, caddr_t); @@ -590,14 +589,14 @@ static int tunoutput( struct ifnet *ifp, struct mbuf *m0, - struct sockaddr *dst, - struct rtentry *rt) -{ + struct route *ro) +{ struct tun_softc *tp = ifp->if_softc; u_short cached_tun_flags; int error; u_int32_t af; - + struct sockaddr *dst = &ro->ro_dst; + TUNDEBUG (ifp, "tunoutput\n"); #ifdef MAC Modified: user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/if_var.h Wed Mar 4 20:58:22 2009 (r189372) @@ -70,6 +70,7 @@ struct socket; struct ether_header; struct carp_if; struct ifvlantrunk; +struct route; #endif #include /* get TAILQ macros */ @@ -148,8 +149,7 @@ struct ifnet { int if_amcount; /* number of all-multicast requests */ /* procedure handles */ int (*if_output) /* output routine (enqueue) */ - (struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); + (struct ifnet *, struct mbuf *, struct route *); void (*if_input) /* input routine (from h/w driver) */ (struct ifnet *, struct mbuf *); void (*if_start) /* initiate output routine */ Modified: user/kmacy/HEAD_fast_net_merge/sys/net/iso88025.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/iso88025.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/iso88025.h Wed Mar 4 20:58:22 2009 (r189372) @@ -165,8 +165,7 @@ struct iso88025_addr { void iso88025_ifattach (struct ifnet *, const u_int8_t *, int); void iso88025_ifdetach (struct ifnet *, int); int iso88025_ioctl (struct ifnet *, int , caddr_t ); -int iso88025_output (struct ifnet *, struct mbuf *, struct sockaddr *, - struct rtentry *); +int iso88025_output (struct ifnet *, struct mbuf *, struct route *ro); void iso88025_input (struct ifnet *, struct mbuf *); #endif Modified: user/kmacy/HEAD_fast_net_merge/sys/net/route.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net/route.h Wed Mar 4 20:58:22 2009 (r189372) @@ -41,9 +41,10 @@ */ /* - * A route consists of a destination address and a reference - * to a routing entry. These are often held by protocols - * in their control blocks, e.g. inpcb. + * A route consists of a destination address, a reference + * to a routing entry, and a reference to an llentry. + * These are often held by protocols in their control + * blocks, e.g. inpcb. */ struct route { struct sockaddr ro_dst; Modified: user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211.c Wed Mar 4 20:58:22 2009 (r189372) @@ -216,8 +216,7 @@ null_update_promisc(struct ifnet *ifp) } static int -null_output(struct ifnet *ifp, struct mbuf *m, - struct sockaddr *dst, struct rtentry *rt0) +null_output(struct ifnet *ifp, struct mbuf *m, struct route *ro) { if_printf(ifp, "discard raw packet\n"); m_freem(m); Modified: user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211_output.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211_output.c Wed Mar 4 20:54:42 2009 (r189371) +++ user/kmacy/HEAD_fast_net_merge/sys/net80211/ieee80211_output.c Wed Mar 4 20:58:22 2009 (r189372) @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -289,13 +290,13 @@ ieee80211_start(struct ifnet *ifp) * will go away when the virtual ap support comes in. */ int *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 21:22:40 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 584971065672; Wed, 4 Mar 2009 21:22:40 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 46CC58FC14; Wed, 4 Mar 2009 21:22:40 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n24LMd0U053651; Wed, 4 Mar 2009 21:22:39 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n24LMdO3053650; Wed, 4 Mar 2009 21:22:39 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903042122.n24LMdO3053650@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 21:22:39 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189374 - user/kmacy/HEAD_fast_net_merge/sys/netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 21:22:40 -0000 Author: kmacy Date: Wed Mar 4 21:22:39 2009 New Revision: 189374 URL: http://svn.freebsd.org/changeset/base/189374 Log: Use per-cpu callouts for tcp_timer 186694, 187660 186694: - convert tcp_timer_activate over to using per-cpu callouts - don't acquire the tcbinfo lock exclusively in tcp_timer_rexmt unless needed for tcp_drop 187660: - mp_maxid may not be valid ensure that we re-schedule on cpuid less than or equal to the current one for tcp callouts Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c Wed Mar 4 21:04:52 2009 (r189373) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c Wed Mar 4 21:22:39 2009 (r189374) @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -118,6 +119,8 @@ int tcp_maxpersistidle; /* max idle time in persist */ int tcp_maxidle; +#define INP_CPU(inp) min(curcpu, ((inp)->inp_flowid % mp_maxid)) + /* * Tcp protocol timeout routine called every 500 ms. * Updates timestamps used for TCP @@ -255,8 +258,8 @@ tcp_timer_2msl(void *xtp) } else { if (tp->t_state != TCPS_TIME_WAIT && (ticks - tp->t_rcvtime) <= tcp_maxidle) - callout_reset(&tp->t_timers->tt_2msl, tcp_keepintvl, - tcp_timer_2msl, tp); + callout_reset_on(&tp->t_timers->tt_2msl, tcp_keepintvl, + tcp_timer_2msl, tp, INP_CPU(inp)); else tp = tcp_close(tp); } @@ -340,9 +343,9 @@ tcp_timer_keep(void *xtp) tp->rcv_nxt, tp->snd_una - 1, 0); free(t_template, M_TEMP); } - callout_reset(&tp->t_timers->tt_keep, tcp_keepintvl, tcp_timer_keep, tp); + callout_reset_on(&tp->t_timers->tt_keep, tcp_keepintvl, tcp_timer_keep, tp, INP_CPU(inp)); } else - callout_reset(&tp->t_timers->tt_keep, tcp_keepidle, tcp_timer_keep, tp); + callout_reset_on(&tp->t_timers->tt_keep, tcp_keepidle, tcp_timer_keep, tp, INP_CPU(inp)); #ifdef TCPDEBUG if (inp->inp_socket->so_options & SO_DEBUG) @@ -447,15 +450,13 @@ tcp_timer_rexmt(void * xtp) CURVNET_SET(tp->t_vnet); INIT_VNET_INET(tp->t_vnet); int rexmt; - int headlocked; struct inpcb *inp; #ifdef TCPDEBUG int ostate; ostate = tp->t_state; #endif - INP_INFO_WLOCK(&V_tcbinfo); - headlocked = 1; + INP_INFO_RLOCK(&V_tcbinfo); inp = tp->t_inpcb; /* * XXXRW: While this assert is in fact correct, bugs in the tcpcb @@ -466,15 +467,16 @@ tcp_timer_rexmt(void * xtp) */ if (inp == NULL) { tcp_timer_race++; - INP_INFO_WUNLOCK(&V_tcbinfo); + INP_INFO_RUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); return; } INP_WLOCK(inp); - if ((inp->inp_vflag & INP_DROPPED) || callout_pending(&tp->t_timers->tt_rexmt) + if ((inp->inp_vflag & INP_DROPPED) + || callout_pending(&tp->t_timers->tt_rexmt) || !callout_active(&tp->t_timers->tt_rexmt)) { + INP_INFO_RUNLOCK(&V_tcbinfo); INP_WUNLOCK(inp); - INP_INFO_WUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); return; } @@ -488,12 +490,22 @@ tcp_timer_rexmt(void * xtp) if (++tp->t_rxtshift > TCP_MAXRXTSHIFT) { tp->t_rxtshift = TCP_MAXRXTSHIFT; V_tcpstat.tcps_timeoutdrop++; + in_pcbref(inp); + INP_INFO_RUNLOCK(&V_tcbinfo); + INP_WUNLOCK(inp); + INP_INFO_WLOCK(&V_tcbinfo); + INP_WLOCK(inp); + if (in_pcbrele(inp)) { + INP_INFO_WUNLOCK(&V_tcbinfo); + CURVNET_RESTORE(); + return; + } tp = tcp_drop(tp, tp->t_softerror ? tp->t_softerror : ETIMEDOUT); + INP_INFO_WUNLOCK(&V_tcbinfo); goto out; } - INP_INFO_WUNLOCK(&V_tcbinfo); - headlocked = 0; + INP_INFO_RUNLOCK(&V_tcbinfo); if (tp->t_rxtshift == 1) { /* * first retransmit; record ssthresh and cwnd so they can @@ -598,8 +610,6 @@ out: #endif if (tp != NULL) INP_WUNLOCK(inp); - if (headlocked) - INP_INFO_WUNLOCK(&V_tcbinfo); CURVNET_RESTORE(); } @@ -608,6 +618,8 @@ tcp_timer_activate(struct tcpcb *tp, int { struct callout *t_callout; void *f_callout; + struct inpcb *inp = tp->t_inpcb; + int cpu = INP_CPU(inp); switch (timer_type) { case TT_DELACK: @@ -636,7 +648,7 @@ tcp_timer_activate(struct tcpcb *tp, int if (delta == 0) { callout_stop(t_callout); } else { - callout_reset(t_callout, delta, f_callout, tp); + callout_reset_on(t_callout, delta, f_callout, tp, cpu); } } From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 21:40:08 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id A38A8106564A; Wed, 4 Mar 2009 21:40:08 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 91A938FC1A; Wed, 4 Mar 2009 21:40:08 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n24Le8EK054074; Wed, 4 Mar 2009 21:40:08 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n24Le8PS054073; Wed, 4 Mar 2009 21:40:08 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903042140.n24Le8PS054073@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 21:40:08 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189376 - user/kmacy/HEAD_fast_net_merge/sys/netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 21:40:08 -0000 Author: kmacy Date: Wed Mar 4 21:40:08 2009 New Revision: 189376 URL: http://svn.freebsd.org/changeset/base/189376 Log: : Add ip_output_fast for handling ip transmit in the common case: 186620, 186624 186620: - Add ip_output_fast for handling ip transmit in the common case 186624: - attempt to further streamline ip_output checks Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 21:38:16 2009 (r189375) +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/ip_output.c Wed Mar 4 21:40:08 2009 (r189376) @@ -117,6 +117,90 @@ static void ip_mloopback extern struct protosw inetsw[]; extern struct flowtable *ipv4_ft; +static int +ip_output_fast(struct mbuf *m, struct route *ro, int flags, int hlen, + struct inpcb *inp) +{ + struct ifnet *ifp = NULL; /* keep compiler happy */ + struct ip *ip; + struct in_ifaddr *ia = NULL; + int sw_csum; + + ia = ifatoia(ro->ro_rt->rt_ifa); + ifp = ro->ro_rt->rt_ifp; + ro->ro_rt->rt_rmx.rmx_pksent++; + + ip = mtod(m, struct ip *); + /* + * If the source address is not specified yet, use the address + * of the outoing interface. + */ + if (ip->ip_src.s_addr == INADDR_ANY) { + /* Interface may have no addresses. */ + if (ia != NULL) { + ip->ip_src = IA_SIN(ia)->sin_addr; + } + } + +#ifdef IPSEC + switch(ip_ipsec_output(&m, inp, &flags, &error, &ro, &iproute, &dst, &ia, &ifp)) { + case 1: + m_freem(m); + return (0); + case -1: + return (0); + case 0: + default: + break; /* Continue with packet processing. */ + } + /* Update variables that are affected by ipsec4_output(). */ + ip = mtod(m, struct ip *); + hlen = ip->ip_hl << 2; +#endif /* IPSEC */ + + m->m_pkthdr.csum_flags |= CSUM_IP; + sw_csum = m->m_pkthdr.csum_flags & ~ifp->if_hwassist; + if (sw_csum & CSUM_DELAY_DATA) { + in_delayed_cksum(m); + sw_csum &= ~CSUM_DELAY_DATA; + } + m->m_pkthdr.csum_flags &= ifp->if_hwassist; + + ip->ip_len = htons(ip->ip_len); + ip->ip_off = htons(ip->ip_off); + ip->ip_sum = 0; + if (sw_csum & CSUM_DELAY_IP) + ip->ip_sum = in_cksum(m, hlen); + + /* + * Record statistics for this interface address. + * With CSUM_TSO the byte/packet count will be slightly + * incorrect because we count the IP+TCP headers only + * once instead of for every generated packet. + */ + if (!(flags & IP_FORWARDING) && ia) { + if (m->m_pkthdr.csum_flags & CSUM_TSO) + ia->ia_ifa.if_opackets += + m->m_pkthdr.len / m->m_pkthdr.tso_segsz; + else + ia->ia_ifa.if_opackets++; + ia->ia_ifa.if_obytes += m->m_pkthdr.len; + } +#ifdef MBUF_STRESS_TEST + if (mbuf_frag_size && m->m_pkthdr.len > mbuf_frag_size) + m = m_fragment(m, M_DONTWAIT, mbuf_frag_size); +#endif + /* + * Reset layer specific mbuf flags + * to avoid confusing lower layers. + */ + m->m_flags &= ~(M_PROTOFLAGS); + + return ((*ifp->if_output)(ifp, m, ro)); +} + +#define IP_RT_SET 0x1 +#define IP_LLE_SET 0x2 /* * IP output. The packet in mbuf chain m contains a skeletal IP * header (with len, off, ttl, proto, tos, src, dst). @@ -134,16 +218,16 @@ ip_output(struct mbuf *m, struct mbuf *o struct ip *ip; struct ifnet *ifp = NULL; /* keep compiler happy */ struct mbuf *m0; + struct in_ifaddr *ia = NULL; int hlen = sizeof (struct ip); - int mtu; + int mtu, flerror = ENOENT; int len, error = 0; int neednewroute = 0, neednewlle = 0, nortfree = 0; struct sockaddr_in *dst = NULL; /* keep compiler happy */ - struct in_ifaddr *ia = NULL; int isbroadcast, sw_csum; + int stateflags = 0; struct route iproute; struct in_addr odst; - struct sockaddr_in *sin; #ifdef IPFIREWALL_FORWARD struct m_tag *fwd_tag = NULL; #endif @@ -151,39 +235,15 @@ ip_output(struct mbuf *m, struct mbuf *o if (ro == NULL) { ro = &iproute; - bzero(ro, sizeof (*ro)); + ro->ro_rt = NULL; + ro->ro_lle = NULL; } if (inp != NULL) { M_SETFIB(m, inp->inp_inc.inc_fibnum); INP_LOCK_ASSERT(inp); - if (inp->inp_flags & (INP_HW_FLOWID|INP_SW_FLOWID)) { - m->m_pkthdr.flowid = inp->inp_flowid; - m->m_flags |= M_FLOWID; - } - if ((ro == &iproute) && (inp->inp_vflag & INP_RT_VALID)) { - if (inp->inp_rt->rt_flags & RTF_UP) { - sin = (struct sockaddr_in *)&ro->ro_dst; - sin->sin_family = AF_INET; - sin->sin_len = sizeof(struct sockaddr_in); - sin->sin_addr.s_addr = inp->inp_faddr.s_addr; - ro->ro_rt = inp->inp_rt; - } else - neednewroute = 1; - } - if ((ro == &iproute) && (inp->inp_flags & INP_LLE_VALID)) { - if (inp->inp_lle->la_flags & LLE_VALID) { - ro->ro_lle = inp->inp_lle; - } else - neednewlle = 1; - } - } - if ((ro == &iproute) && (ro->ro_rt == NULL) && (ro->ro_lle == NULL)) { - if (flowtable_lookup(ipv4_ft, m, ro) == 0) - nortfree = 1; } - if (opt) { len = 0; m = ip_insertoptions(m, opt, &len); @@ -214,6 +274,85 @@ ip_output(struct mbuf *m, struct mbuf *o dst = (struct sockaddr_in *)&ro->ro_dst; again: +#ifdef IPFIREWALL_FORWARD + if (ro->ro_rt == NULL && fwd_tag == NULL) { +#else + if (ro->ro_rt == NULL) { +#endif + bzero(dst, sizeof(*dst)); + dst->sin_family = AF_INET; + dst->sin_len = sizeof(*dst); + dst->sin_addr = ip->ip_dst; + } + if (ro != &iproute) { + if (ro->ro_rt != NULL && (ro->ro_rt->rt_flags & RTF_UP)) + stateflags |= IP_RT_SET; + if (ro->ro_lle != NULL && (ro->ro_lle->la_flags & LLE_VALID)) + stateflags |= IP_LLE_SET; + goto skipcachecheck; + } + + if (inp != NULL) { + if (inp->inp_flags & (INP_HW_FLOWID|INP_SW_FLOWID)) { + m->m_pkthdr.flowid = inp->inp_flowid; + m->m_flags |= M_FLOWID; + } + if (inp->inp_vflag & INP_RT_VALID) { + if (inp->inp_rt->rt_flags & RTF_UP) { + ro->ro_rt = inp->inp_rt; + stateflags |= IP_RT_SET; + } else + neednewroute = 1; + } + if (inp->inp_flags & INP_LLE_VALID) { + if (inp->inp_lle->la_flags & LLE_VALID) { + ro->ro_lle = inp->inp_lle; + stateflags |= IP_LLE_SET; + } else + neednewlle = 1; + } + } + if ((stateflags & (IP_LLE_SET|IP_RT_SET)) != (IP_LLE_SET|IP_RT_SET)) { + if ((flerror = flowtable_lookup(ipv4_ft, m, ro)) == 0) { + nortfree = 1; + stateflags |= IP_RT_SET; + if (ro->ro_lle != NULL) + stateflags |= IP_LLE_SET; + } + } +skipcachecheck: + + /* + * Check if we can take a huge shortcut + */ + if ((stateflags & (IP_LLE_SET|IP_RT_SET)) == (IP_LLE_SET|IP_RT_SET) + && (flags & (IP_SENDONES|IP_ROUTETOIF)) == 0 + && !PFIL_HOOKED(&inet_pfil_hook) + && !IN_MULTICAST(ntohl(ip->ip_dst.s_addr))) { + ifp = ro->ro_rt->rt_ifp; + + dst = (struct sockaddr_in *)ro->ro_rt->rt_gateway; + if (ro->ro_rt->rt_flags & RTF_GATEWAY) + bcopy(ro->ro_rt->rt_gateway, &ro->ro_dst, + sizeof(struct sockaddr)); + + if (ro->ro_rt->rt_flags & RTF_HOST) { + if (ro->ro_rt->rt_rmx.rmx_mtu > ifp->if_mtu) + ro->ro_rt->rt_rmx.rmx_mtu = ifp->if_mtu; + mtu = ro->ro_rt->rt_rmx.rmx_mtu; + isbroadcast = (ro->ro_rt->rt_flags & RTF_BROADCAST); + } else { + isbroadcast = in_broadcast(dst->sin_addr, ifp); + mtu = ifp->if_mtu; + } + + if (!isbroadcast && (ip->ip_len <= mtu || + (m->m_pkthdr.csum_flags & ifp->if_hwassist & CSUM_TSO) != 0 + || ((ip->ip_off & IP_DF) == 0 + && (ifp->if_hwassist & CSUM_FRAGMENT)))) + return (ip_output_fast(m, ro, flags, hlen, inp)); + } + /* * If there is a cached route, * check that it is to the same destination @@ -221,24 +360,20 @@ again: * The address family should also be checked in case of sharing the * cache with IPv6. */ - if (ro->ro_rt && ((ro->ro_rt->rt_flags & RTF_UP) == 0 || + if (ro->ro_rt != NULL && ((ro->ro_rt->rt_flags & RTF_UP) == 0 || dst->sin_family != AF_INET || dst->sin_addr.s_addr != ip->ip_dst.s_addr)) { - if ((nortfree == 0) && - (inp == NULL || (ro->ro_rt != inp->inp_rt))) + /* + * XXX need to check that ro->ro_rt isn't coming from a flow + * lookup in ip_forward ... perhaps pass in a new flag + * + */ + if (((inp->inp_vflag & INP_RT_VALID) == 0) && + flerror != 0) RTFREE(ro->ro_rt); - ro->ro_rt = (struct rtentry *)NULL; - } -#ifdef IPFIREWALL_FORWARD - if (ro->ro_rt == NULL && fwd_tag == NULL) { -#else - if (ro->ro_rt == NULL) { -#endif - bzero(dst, sizeof(*dst)); - dst->sin_family = AF_INET; - dst->sin_len = sizeof(*dst); - dst->sin_addr = ip->ip_dst; + ro->ro_rt = NULL; } + /* * If routing to interface only, short circuit routing lookup. * The use of an all-ones broadcast address implies this; an From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 22:22:31 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id F31C9106564A; Wed, 4 Mar 2009 22:22:30 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id E195A8FC1D; Wed, 4 Mar 2009 22:22:30 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n24MMUgk055025; Wed, 4 Mar 2009 22:22:30 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n24MMUrF055023; Wed, 4 Mar 2009 22:22:30 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903042222.n24MMUrF055023@svn.freebsd.org> From: Kip Macy Date: Wed, 4 Mar 2009 22:22:30 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189378 - user/kmacy/HEAD_fast_net_merge/sys/dev/e1000 X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 22:22:31 -0000 Author: kmacy Date: Wed Mar 4 22:22:30 2009 New Revision: 189378 URL: http://svn.freebsd.org/changeset/base/189378 Log: add option to make em use a buf_ring 186584 186584: - Bring in if_transmit support for if_em - fix a use after free bug that was fixed in r185621 Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.c user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.h Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.c ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.c Wed Mar 4 22:05:25 2009 (r189377) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.c Wed Mar 4 22:22:30 2009 (r189378) @@ -39,6 +39,7 @@ #include #include +#include #include #include #include @@ -889,6 +890,7 @@ em_detach(device_t dev) em_free_pci_resources(adapter); bus_generic_detach(dev); if_free(ifp); + drbr_free(adapter->br, M_DEVBUF); em_free_transmit_structures(adapter); em_free_receive_structures(adapter); @@ -984,6 +986,81 @@ em_resume(device_t dev) * the packet is requeued. **********************************************************************/ +#ifdef IFNET_MULTIQUEUE +static int +em_transmit_locked(struct ifnet *ifp, struct mbuf *m) +{ + struct adapter *adapter = ifp->if_softc; + int error; + + EM_TX_LOCK_ASSERT(adapter); + if (((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + || (!adapter->link_active)) { + error = drbr_enqueue(ifp, adapter->br, m); + return (error); + } + + if (buf_ring_empty(adapter->br) && + (adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD)) { + if (em_xmit(adapter, &m)) { + if (m && (error = drbr_enqueue(ifp, adapter->br, m)) != 0) { + return (error); + } + } else{ + /* Send a copy of the frame to the BPF listener */ + ETHER_BPF_MTAP(ifp, m); + } + } else if ((error = drbr_enqueue(ifp, adapter->br, m)) != 0) + return (error); + + if (!buf_ring_empty(adapter->br)) + em_start_locked(ifp); + + return (0); +} + +static void +em_start_locked(struct ifnet *ifp) +{ + struct adapter *adapter = ifp->if_softc; + struct mbuf *m_head; + + EM_TX_LOCK_ASSERT(adapter); + + if ((ifp->if_drv_flags & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + return; + if (!adapter->link_active) + return; + + while ((adapter->num_tx_desc_avail > EM_TX_OP_THRESHOLD) + && (!buf_ring_empty(adapter->br))) { + + m_head = buf_ring_dequeue_sc(adapter->br); + if (m_head == NULL) + break; + /* + * Encapsulation can modify our pointer, and or make it + * NULL on failure. In that event, we can't requeue. + */ + if (em_xmit(adapter, &m_head)) { + if (m_head == NULL) + break; + break; + } + + /* Send a copy of the frame to the BPF listener */ + ETHER_BPF_MTAP(ifp, m_head); + + /* Set timeout in case hardware has problems transmitting. */ + adapter->watchdog_timer = EM_TX_TIMEOUT; + } + if ((adapter->num_tx_desc_avail <= EM_TX_OP_THRESHOLD)) + ifp->if_drv_flags |= IFF_DRV_OACTIVE; + +} +#else static void em_start_locked(struct ifnet *ifp) { @@ -1022,6 +1099,7 @@ em_start_locked(struct ifnet *ifp) adapter->watchdog_timer = EM_TX_TIMEOUT; } } +#endif static void em_start(struct ifnet *ifp) @@ -1034,6 +1112,23 @@ em_start(struct ifnet *ifp) EM_TX_UNLOCK(adapter); } +static int +em_transmit(struct ifnet *ifp, struct mbuf *m) +{ + + struct adapter *adapter = ifp->if_softc; + int error = 0; + + if(EM_TX_TRYLOCK(adapter)) { + if (ifp->if_drv_flags & IFF_DRV_RUNNING) + error = em_transmit_locked(ifp, m); + EM_TX_UNLOCK(adapter); + } else + error = drbr_enqueue(ifp, adapter->br, m); + + return (error); +} + /********************************************************************* * Ioctl entry point * @@ -1597,7 +1692,11 @@ em_poll(struct ifnet *ifp, enum poll_cmd EM_TX_LOCK(adapter); em_txeof(adapter); +#ifdef IFNET_MULTIQUEUE + if (!buf_ring_empty(adapter->br)) +#else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) +#endif em_start_locked(ifp); EM_TX_UNLOCK(adapter); } @@ -1665,8 +1764,15 @@ em_intr(void *arg) } EM_CORE_UNLOCK(adapter); + if (ifp->if_drv_flags & IFF_DRV_RUNNING && - !IFQ_DRV_IS_EMPTY(&ifp->if_snd)) +#ifdef IFNET_MULTIQUEUE + !buf_ring_empty(adapter->br) +#else + !IFQ_DRV_IS_EMPTY(&ifp->if_snd) +#endif + ) + em_start(ifp); } @@ -1705,7 +1811,11 @@ em_handle_rxtx(void *context, int pendin EM_TX_LOCK(adapter); em_txeof(adapter); +#ifdef IFNET_MULTIQUEUE + if (!buf_ring_empty(adapter->br)) +#else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) +#endif em_start_locked(ifp); EM_TX_UNLOCK(adapter); } @@ -1732,9 +1842,19 @@ em_handle_tx(void *context, int pending) struct ifnet *ifp = adapter->ifp; if (ifp->if_drv_flags & IFF_DRV_RUNNING) { +#ifdef IFNET_MULTIQUEUE + if (!EM_TX_TRYLOCK(adapter)) + return; +#else EM_TX_LOCK(adapter); +#endif + em_txeof(adapter); +#ifdef IFNET_MULTIQUEUE + if (!buf_ring_empty(adapter->br)) +#else if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) +#endif em_start_locked(ifp); EM_TX_UNLOCK(adapter); } @@ -2083,22 +2203,14 @@ em_xmit(struct adapter *adapter, struct error = bus_dmamap_load_mbuf_sg(adapter->txtag, map, *m_headp, segs, &nsegs, BUS_DMA_NOWAIT); - if (error == ENOMEM) { - adapter->no_tx_dma_setup++; - return (error); - } else if (error != 0) { + if (error) { adapter->no_tx_dma_setup++; m_freem(*m_headp); *m_headp = NULL; return (error); } - } else if (error == ENOMEM) { - adapter->no_tx_dma_setup++; - return (error); } else if (error != 0) { adapter->no_tx_dma_setup++; - m_freem(*m_headp); - *m_headp = NULL; return (error); } @@ -2531,7 +2643,8 @@ em_local_timer(void *arg) struct ifnet *ifp = adapter->ifp; EM_CORE_LOCK_ASSERT(adapter); - + taskqueue_enqueue(adapter->tq, + &adapter->rxtx_task); em_update_link_status(adapter); em_update_stats_counters(adapter); @@ -3125,7 +3238,11 @@ em_setup_interface(device_t dev, struct ether_ifattach(ifp, adapter->hw.mac.addr); ifp->if_capabilities = ifp->if_capenable = 0; - +#ifdef IFNET_MULTIQUEUE + ifp->if_transmit = em_transmit; + adapter->br = buf_ring_alloc(2048, M_DEVBUF, M_WAITOK, &adapter->tx_mtx); +#endif + if (adapter->hw.mac.type >= e1000_82543) { int version_cap; #if __FreeBSD_version < 700000 Modified: user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.h ============================================================================== --- user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.h Wed Mar 4 22:05:25 2009 (r189377) +++ user/kmacy/HEAD_fast_net_merge/sys/dev/e1000/if_em.h Wed Mar 4 22:22:30 2009 (r189378) @@ -36,6 +36,7 @@ #ifndef _EM_H_DEFINED_ #define _EM_H_DEFINED_ +#define IFNET_MULTIQUEUE /* Tunables */ /* @@ -301,6 +302,9 @@ struct em_dma_alloc { /* Our adapter structure */ struct adapter { struct ifnet *ifp; +#ifdef IFNET_MULTIQUEUE + struct buf_ring *br; +#endif struct e1000_hw hw; /* FreeBSD operating-system-specific structures. */ @@ -482,6 +486,7 @@ typedef struct _DESCRIPTOR_PAIR #define EM_RX_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->rx_mtx) #define EM_CORE_LOCK(_sc) mtx_lock(&(_sc)->core_mtx) #define EM_TX_LOCK(_sc) mtx_lock(&(_sc)->tx_mtx) +#define EM_TX_TRYLOCK(_sc) mtx_trylock(&(_sc)->tx_mtx) #define EM_RX_LOCK(_sc) mtx_lock(&(_sc)->rx_mtx) #define EM_CORE_UNLOCK(_sc) mtx_unlock(&(_sc)->core_mtx) #define EM_TX_UNLOCK(_sc) mtx_unlock(&(_sc)->tx_mtx) From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 22:59:59 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id D1AAC106564A; Wed, 4 Mar 2009 22:59:59 +0000 (UTC) (envelope-from jhb@freebsd.org) Received: from cyrus.watson.org (cyrus.watson.org [65.122.17.42]) by mx1.freebsd.org (Postfix) with ESMTP id A20728FC0A; Wed, 4 Mar 2009 22:59:59 +0000 (UTC) (envelope-from jhb@freebsd.org) Received: from server.baldwin.cx (pool-98-109-39-197.nwrknj.fios.verizon.net [98.109.39.197]) by cyrus.watson.org (Postfix) with ESMTPSA id 1A50646B3C; Wed, 4 Mar 2009 17:59:59 -0500 (EST) Received: from localhost (john@localhost [127.0.0.1]) (authenticated bits=0) by server.baldwin.cx (8.14.3/8.14.3) with ESMTP id n24Mxfsm098592; Wed, 4 Mar 2009 17:59:53 -0500 (EST) (envelope-from jhb@freebsd.org) From: John Baldwin To: Kip Macy Date: Wed, 4 Mar 2009 17:59:31 -0500 User-Agent: KMail/1.9.7 References: <200903042122.n24LMdO3053650@svn.freebsd.org> In-Reply-To: <200903042122.n24LMdO3053650@svn.freebsd.org> MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit Content-Disposition: inline Message-Id: <200903041759.31535.jhb@freebsd.org> X-Greylist: Sender succeeded SMTP AUTH authentication, not delayed by milter-greylist-2.0.2 (server.baldwin.cx [127.0.0.1]); Wed, 04 Mar 2009 17:59:53 -0500 (EST) X-Virus-Scanned: ClamAV 0.94.2/9068/Wed Mar 4 16:56:13 2009 on server.baldwin.cx X-Virus-Status: Clean X-Spam-Status: No, score=-4.4 required=4.2 tests=ALL_TRUSTED,AWL,BAYES_00 autolearn=ham version=3.1.3 X-Spam-Checker-Version: SpamAssassin 3.1.3 (2006-06-01) on server.baldwin.cx Cc: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: Re: svn commit: r189374 - user/kmacy/HEAD_fast_net_merge/sys/netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 23:00:00 -0000 On Wednesday 04 March 2009 4:22:39 pm Kip Macy wrote: > Author: kmacy > Date: Wed Mar 4 21:22:39 2009 > New Revision: 189374 > URL: http://svn.freebsd.org/changeset/base/189374 > > Log: > Use per-cpu callouts for tcp_timer > 186694, 187660 > > 186694: > - convert tcp_timer_activate over to using > per-cpu callouts > - don't acquire the tcbinfo lock exclusively > in tcp_timer_rexmt unless needed for tcp_drop > > 187660: > - mp_maxid may not be valid ensure that we > re-schedule on cpuid less than or equal to > the current one for tcp callouts > > Modified: > user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c > > Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c > ============================================================================== > --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c Wed Mar 4 21:04:52 2009 (r189373) > +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c Wed Mar 4 21:22:39 2009 (r189374) > @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); > #include > #include > #include > +#include > #include > #include > #include > @@ -118,6 +119,8 @@ int tcp_maxpersistidle; > /* max idle time in persist */ > int tcp_maxidle; > > +#define INP_CPU(inp) min(curcpu, ((inp)->inp_flowid % mp_maxid)) This is not really safe. CPU ID's may be sparse. The only guarantees you have are that 0 is the boot CPU, and that all valid CPU IDs are in the range [0 .. mp_maxid] (inclusive). Thus, you could have a system that only has CPUs 0 and 3 and if you are on CPU 3 and flowid is 5, then this will choose min(3, 5 % 3) == min(3, 2) == 2 which is an invalid CPU. -- John Baldwin From owner-svn-src-user@FreeBSD.ORG Wed Mar 4 23:46:23 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 8C123106564A for ; Wed, 4 Mar 2009 23:46:23 +0000 (UTC) (envelope-from mat.macy@gmail.com) Received: from rv-out-0506.google.com (rv-out-0506.google.com [209.85.198.239]) by mx1.freebsd.org (Postfix) with ESMTP id 5DAE88FC17 for ; Wed, 4 Mar 2009 23:46:23 +0000 (UTC) (envelope-from mat.macy@gmail.com) Received: by rv-out-0506.google.com with SMTP id f6so4062798rvb.43 for ; Wed, 04 Mar 2009 15:46:23 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; h=domainkey-signature:mime-version:sender:received:in-reply-to :references:date:x-google-sender-auth:message-id:subject:from:to:cc :content-type:content-transfer-encoding; bh=ZS8+bqyZ7CS19NqRVGu46mdsbo3/XoVi8K3Rhs7cpsM=; b=oUQ0cOfNcw9Op110GwdmmOcsb3OTKQ6xfo6KdZpXMUf0XIN54P/MNNtFcujOttFls2 m9IMJj4FXfcEcGgRMyI7q7uqGvpRw0oqw30oFDpFNt7FwKTAqK50kTSpPiFMfV3dd0kz BQ9rSo2yN2Q4qdOXX9ltoNgistl3cFkJwtqTY= DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; h=mime-version:sender:in-reply-to:references:date :x-google-sender-auth:message-id:subject:from:to:cc:content-type :content-transfer-encoding; b=iTHnu2GLDw0CUAAL5+xMtnaVvAGsHLm7yyI8m12pkItiJrQRhF/GQb1MOqmAbQjyUI bTCcNlOheGv8jkbiSfE1Ox0xtmaSXQhXr+3GZX5nkivZ1c2s/pxpszyQEvTkr463podH uJyeRh97OljqY+YH2fVhDFowGS6QHTOlDQUtQ= MIME-Version: 1.0 Sender: mat.macy@gmail.com Received: by 10.141.52.3 with SMTP id e3mr225494rvk.73.1236208855113; Wed, 04 Mar 2009 15:20:55 -0800 (PST) In-Reply-To: <200903041759.31535.jhb@freebsd.org> References: <200903042122.n24LMdO3053650@svn.freebsd.org> <200903041759.31535.jhb@freebsd.org> Date: Wed, 4 Mar 2009 15:20:55 -0800 X-Google-Sender-Auth: 4bf395658eedf924 Message-ID: <3c1674c90903041520o145b0c42uc1dd2d41870432e6@mail.gmail.com> From: Kip Macy To: John Baldwin Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable Cc: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: Re: svn commit: r189374 - user/kmacy/HEAD_fast_net_merge/sys/netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 04 Mar 2009 23:46:23 -0000 Do we have any mechanism for handling sparse cpuids? If so I can do modulo the number of active cpus and then taking that value as an index. If not its a bit of an academic observation. -Kip On Wed, Mar 4, 2009 at 2:59 PM, John Baldwin wrote: > On Wednesday 04 March 2009 4:22:39 pm Kip Macy wrote: >> Author: kmacy >> Date: Wed Mar =A04 21:22:39 2009 >> New Revision: 189374 >> URL: http://svn.freebsd.org/changeset/base/189374 >> >> Log: >> =A0 Use per-cpu callouts for tcp_timer >> =A0 186694, 187660 >> >> =A0 186694: >> =A0 - convert tcp_timer_activate over to using >> =A0 =A0 per-cpu callouts >> =A0 - don't acquire the tcbinfo lock exclusively >> =A0 =A0 in tcp_timer_rexmt unless needed for tcp_drop >> >> =A0 187660: >> =A0 - mp_maxid may not be valid ensure that we >> =A0 =A0 re-schedule on cpuid less than or equal to >> =A0 =A0 the current one for tcp callouts >> >> Modified: >> =A0 user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c >> >> Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c >> > =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c =A0 =A0Wed Ma= r =A04 > 21:04:52 2009 =A0 (r189373) >> +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c =A0 =A0Wed Ma= r =A04 > 21:22:39 2009 =A0 (r189374) >> @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); >> =A0#include >> =A0#include >> =A0#include >> +#include >> =A0#include >> =A0#include >> =A0#include >> @@ -118,6 +119,8 @@ int =A0 =A0 =A0 tcp_maxpersistidle; >> =A0 =A0 =A0 /* max idle time in persist */ >> =A0int =A0tcp_maxidle; >> >> +#define =A0 =A0 =A0INP_CPU(inp) =A0 =A0min(curcpu, ((inp)->inp_flowid %= mp_maxid)) > > This is not really safe. =A0CPU ID's may be sparse. =A0The only guarantee= s you > have are that 0 is the boot CPU, and that all valid CPU IDs are in the ra= nge > [0 .. mp_maxid] (inclusive). =A0Thus, you could have a system that only h= as > CPUs 0 and 3 and if you are on CPU 3 and flowid is 5, then this will choo= se > min(3, 5 % 3) =3D=3D min(3, 2) =3D=3D 2 which is an invalid CPU. > > -- > John Baldwin > From owner-svn-src-user@FreeBSD.ORG Thu Mar 5 15:16:13 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 798831065700; Thu, 5 Mar 2009 15:16:13 +0000 (UTC) (envelope-from jhb@freebsd.org) Received: from cyrus.watson.org (cyrus.watson.org [65.122.17.42]) by mx1.freebsd.org (Postfix) with ESMTP id 37C4C8FC24; Thu, 5 Mar 2009 15:16:13 +0000 (UTC) (envelope-from jhb@freebsd.org) Received: from server.baldwin.cx (pool-98-109-39-197.nwrknj.fios.verizon.net [98.109.39.197]) by cyrus.watson.org (Postfix) with ESMTPSA id B8F5E46B39; Thu, 5 Mar 2009 10:16:12 -0500 (EST) Received: from localhost (john@localhost [127.0.0.1]) (authenticated bits=0) by server.baldwin.cx (8.14.3/8.14.3) with ESMTP id n25FG6sD004602; Thu, 5 Mar 2009 10:16:06 -0500 (EST) (envelope-from jhb@freebsd.org) From: John Baldwin To: Kip Macy Date: Thu, 5 Mar 2009 10:04:36 -0500 User-Agent: KMail/1.9.7 References: <200903042122.n24LMdO3053650@svn.freebsd.org> <200903041759.31535.jhb@freebsd.org> <3c1674c90903041520o145b0c42uc1dd2d41870432e6@mail.gmail.com> In-Reply-To: <3c1674c90903041520o145b0c42uc1dd2d41870432e6@mail.gmail.com> MIME-Version: 1.0 Content-Type: text/plain; charset="iso-8859-1" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline Message-Id: <200903051004.36626.jhb@freebsd.org> X-Greylist: Sender succeeded SMTP AUTH authentication, not delayed by milter-greylist-2.0.2 (server.baldwin.cx [127.0.0.1]); Thu, 05 Mar 2009 10:16:06 -0500 (EST) X-Virus-Scanned: ClamAV 0.94.2/9073/Thu Mar 5 08:50:21 2009 on server.baldwin.cx X-Virus-Status: Clean X-Spam-Status: No, score=-4.4 required=4.2 tests=ALL_TRUSTED,AWL,BAYES_00 autolearn=ham version=3.1.3 X-Spam-Checker-Version: SpamAssassin 3.1.3 (2006-06-01) on server.baldwin.cx Cc: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: Re: svn commit: r189374 - user/kmacy/HEAD_fast_net_merge/sys/netinet X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 05 Mar 2009 15:16:13 -0000 On Wednesday 04 March 2009 6:20:55 pm Kip Macy wrote: > Do we have any mechanism for handling sparse cpuids? > If so I can do modulo the number of active cpus and then taking that > value as an index. If not its a bit of an academic observation. >=20 > -Kip The rest of the kernel is careful to handle sparse cpuids already. You can= =20 use CPU_ABSENT() to see if a CPU is present or not. You cound pre-populate= a=20 table during boot if you wished by doing something like: int active_cpus[MAXCPU]; sysinit_foo() { int i, cpu; i =3D 0; for (cpu =3D 0; cpu <=3D mp_maxid; cpu++) { if (CPU_ABSENT(cpu) continue; active_cpus[i] =3D cpu; i++; } KASSERT(i =3D=3D mp_ncpu, ("bad juju")); } Then you could use (active_cpus[flowid & mp_ncpu]) as a CPU. Eventually wh= en=20 we support removing and adding CPUs at runtime (or disabling/enabling them = at=20 runtime) there will be eventhandlers that you can hook to adjust your=20 active_cpus array. =20 > On Wed, Mar 4, 2009 at 2:59 PM, John Baldwin wrote: > > On Wednesday 04 March 2009 4:22:39 pm Kip Macy wrote: > >> Author: kmacy > >> Date: Wed Mar =A04 21:22:39 2009 > >> New Revision: 189374 > >> URL: http://svn.freebsd.org/changeset/base/189374 > >> > >> Log: > >> =A0 Use per-cpu callouts for tcp_timer > >> =A0 186694, 187660 > >> > >> =A0 186694: > >> =A0 - convert tcp_timer_activate over to using > >> =A0 =A0 per-cpu callouts > >> =A0 - don't acquire the tcbinfo lock exclusively > >> =A0 =A0 in tcp_timer_rexmt unless needed for tcp_drop > >> > >> =A0 187660: > >> =A0 - mp_maxid may not be valid ensure that we > >> =A0 =A0 re-schedule on cpuid less than or equal to > >> =A0 =A0 the current one for tcp callouts > >> > >> Modified: > >> =A0 user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c > >> > >> Modified: user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c > >> > >=20 =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D > >> --- user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c =A0 =A0Wed = Mar =A04 > > 21:04:52 2009 =A0 (r189373) > >> +++ user/kmacy/HEAD_fast_net_merge/sys/netinet/tcp_timer.c =A0 =A0Wed = Mar =A04 > > 21:22:39 2009 =A0 (r189374) > >> @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); > >> =A0#include > >> =A0#include > >> =A0#include > >> +#include > >> =A0#include > >> =A0#include > >> =A0#include > >> @@ -118,6 +119,8 @@ int =A0 =A0 =A0 tcp_maxpersistidle; > >> =A0 =A0 =A0 /* max idle time in persist */ > >> =A0int =A0tcp_maxidle; > >> > >> +#define =A0 =A0 =A0INP_CPU(inp) =A0 =A0min(curcpu, ((inp)->inp_flowid= % mp_maxid)) > > > > This is not really safe. =A0CPU ID's may be sparse. =A0The only guarant= ees you > > have are that 0 is the boot CPU, and that all valid CPU IDs are in the= =20 range > > [0 .. mp_maxid] (inclusive). =A0Thus, you could have a system that only= has > > CPUs 0 and 3 and if you are on CPU 3 and flowid is 5, then this will=20 choose > > min(3, 5 % 3) =3D=3D min(3, 2) =3D=3D 2 which is an invalid CPU. > > > > -- > > John Baldwin > > >=20 =2D-=20 John Baldwin From owner-svn-src-user@FreeBSD.ORG Fri Mar 6 16:47:50 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id A321E106566B; Fri, 6 Mar 2009 16:47:50 +0000 (UTC) (envelope-from dfr@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 90B938FC15; Fri, 6 Mar 2009 16:47:50 +0000 (UTC) (envelope-from dfr@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n26GlooH024129; Fri, 6 Mar 2009 16:47:50 GMT (envelope-from dfr@svn.freebsd.org) Received: (from dfr@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n26GloMR024127; Fri, 6 Mar 2009 16:47:50 GMT (envelope-from dfr@svn.freebsd.org) Message-Id: <200903061647.n26GloMR024127@svn.freebsd.org> From: Doug Rabson Date: Fri, 6 Mar 2009 16:47:50 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189451 - in user/dfr/xenhvm/6/sys: conf dev/xen/balloon X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 06 Mar 2009 16:47:50 -0000 Author: dfr Date: Fri Mar 6 16:47:50 2009 New Revision: 189451 URL: http://svn.freebsd.org/changeset/base/189451 Log: Finish implementing the balloon driver and update it to be closer to current Linux sources. Modified: user/dfr/xenhvm/6/sys/conf/files user/dfr/xenhvm/6/sys/dev/xen/balloon/balloon.c Modified: user/dfr/xenhvm/6/sys/conf/files ============================================================================== --- user/dfr/xenhvm/6/sys/conf/files Fri Mar 6 15:35:37 2009 (r189450) +++ user/dfr/xenhvm/6/sys/conf/files Fri Mar 6 16:47:50 2009 (r189451) @@ -2059,6 +2059,8 @@ xen/xenbus/xenbus_if.m optional xen xen/xenbus/xenbus_probe.c optional xen #xen/xenbus/xenbus_probe_backend.c optional xen xen/xenbus/xenbus_xs.c optional xen +dev/xen/balloon/balloon.c optional xen +dev/xen/balloon/balloon.c optional xenhvm dev/xen/console/console.c optional xen dev/xen/console/xencons_ring.c optional xen dev/xen/blkfront/blkfront.c optional xen Modified: user/dfr/xenhvm/6/sys/dev/xen/balloon/balloon.c ============================================================================== --- user/dfr/xenhvm/6/sys/dev/xen/balloon/balloon.c Fri Mar 6 15:35:37 2009 (r189450) +++ user/dfr/xenhvm/6/sys/dev/xen/balloon/balloon.c Fri Mar 6 16:47:50 2009 (r189451) @@ -34,11 +34,24 @@ __FBSDID("$FreeBSD$"); #include #include +#include +#include +#include #include +#include -#include -#include -#include +#include +#include +#include +#include +#include + +#include +#include + +MALLOC_DEFINE(M_BALLOON, "Balloon", "Xen Balloon Driver"); + +struct mtx balloon_mutex; /* * Protects atomic reservation decrease/increase against concurrent increases. @@ -46,23 +59,44 @@ __FBSDID("$FreeBSD$"); * balloon lists. */ struct mtx balloon_lock; -#ifdef notyet - -/* We aim for 'current allocation' == 'target allocation'. */ -static unsigned long current_pages; -static unsigned long target_pages; -/* VM /proc information for memory */ -extern unsigned long totalram_pages; +/* We increase/decrease in batches which fit in a page */ +static unsigned long frame_list[PAGE_SIZE / sizeof(unsigned long)]; +#define ARRAY_SIZE(A) (sizeof(A) / sizeof(A[0])) + +struct balloon_stats { + /* We aim for 'current allocation' == 'target allocation'. */ + unsigned long current_pages; + unsigned long target_pages; + /* We may hit the hard limit in Xen. If we do then we remember it. */ + unsigned long hard_limit; + /* + * Drivers may alter the memory reservation independently, but they + * must inform the balloon driver so we avoid hitting the hard limit. + */ + unsigned long driver_pages; + /* Number of pages in high- and low-memory balloons. */ + unsigned long balloon_low; + unsigned long balloon_high; +}; -/* We may hit the hard limit in Xen. If we do then we remember it. */ -static unsigned long hard_limit; +static struct balloon_stats balloon_stats; +#define bs balloon_stats -/* - * Drivers may alter the memory reservation independently, but they must - * inform the balloon driver so that we can avoid hitting the hard limit. - */ -static unsigned long driver_pages; +SYSCTL_DECL(_dev_xen); +SYSCTL_NODE(_dev_xen, OID_AUTO, balloon, CTLFLAG_RD, NULL, "Balloon"); +SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, current, CTLFLAG_RD, + &bs.current_pages, 0, "Current allocation"); +SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, target, CTLFLAG_RD, + &bs.target_pages, 0, "Target allocation"); +SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, driver_pages, CTLFLAG_RD, + &bs.driver_pages, 0, "Driver pages"); +SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, hard_limit, CTLFLAG_RD, + &bs.hard_limit, 0, "Xen hard limit"); +SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, low_mem, CTLFLAG_RD, + &bs.balloon_low, 0, "Low-mem balloon"); +SYSCTL_ULONG(_dev_xen_balloon, OID_AUTO, high_mem, CTLFLAG_RD, + &bs.balloon_high, 0, "High-mem balloon"); struct balloon_entry { vm_page_t page; @@ -72,9 +106,6 @@ struct balloon_entry { /* List of ballooned pages, threaded through the mem_map array. */ static STAILQ_HEAD(,balloon_entry) ballooned_pages; -static unsigned long balloon_low, balloon_high; - - /* Main work function, always executed in process context. */ static void balloon_process(void *unused); @@ -89,10 +120,10 @@ balloon_append(vm_page_t page) { struct balloon_entry *entry; - entry = malloc(sizeof(struct balloon_entry), M_WAITOK); - + entry = malloc(sizeof(struct balloon_entry), M_BALLOON, M_WAITOK); + entry->page = page; STAILQ_INSERT_HEAD(&ballooned_pages, entry, list); - balloon_low++; + bs.balloon_low++; } /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */ @@ -111,13 +142,13 @@ balloon_retrieve(void) page = entry->page; free(entry, M_DEVBUF); - balloon_low--; + bs.balloon_low--; return page; } static void -balloon_alarm(unsigned long unused) +balloon_alarm(void *unused) { wakeup(balloon_process); } @@ -125,17 +156,56 @@ balloon_alarm(unsigned long unused) static unsigned long current_target(void) { - unsigned long target = min(target_pages, hard_limit); - if (target > (current_pages + balloon_low + balloon_high)) - target = current_pages + balloon_low + balloon_high; + unsigned long target = min(bs.target_pages, bs.hard_limit); + if (target > (bs.current_pages + bs.balloon_low + bs.balloon_high)) + target = bs.current_pages + bs.balloon_low + bs.balloon_high; return target; } +static unsigned long +minimum_target(void) +{ +#ifdef XENHVM +#define max_pfn physmem +#endif + unsigned long min_pages, curr_pages = current_target(); + +#define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT)) + /* Simple continuous piecewiese linear function: + * max MiB -> min MiB gradient + * 0 0 + * 16 16 + * 32 24 + * 128 72 (1/2) + * 512 168 (1/4) + * 2048 360 (1/8) + * 8192 552 (1/32) + * 32768 1320 + * 131072 4392 + */ + if (max_pfn < MB2PAGES(128)) + min_pages = MB2PAGES(8) + (max_pfn >> 1); + else if (max_pfn < MB2PAGES(512)) + min_pages = MB2PAGES(40) + (max_pfn >> 2); + else if (max_pfn < MB2PAGES(2048)) + min_pages = MB2PAGES(104) + (max_pfn >> 3); + else + min_pages = MB2PAGES(296) + (max_pfn >> 5); +#undef MB2PAGES + + /* Don't enforce growth */ + return min(min_pages, curr_pages); +#ifndef CONFIG_XEN +#undef max_pfn +#endif +} + static int increase_reservation(unsigned long nr_pages) { - unsigned long *mfn_list, pfn, i, flags; - struct page *page; + unsigned long pfn, i; + struct balloon_entry *entry; + vm_page_t page; long rc; struct xen_memory_reservation reservation = { .address_bits = 0, @@ -143,64 +213,81 @@ increase_reservation(unsigned long nr_pa .domid = DOMID_SELF }; - if (nr_pages > (PAGE_SIZE / sizeof(unsigned long))) - nr_pages = PAGE_SIZE / sizeof(unsigned long); + if (nr_pages > ARRAY_SIZE(frame_list)) + nr_pages = ARRAY_SIZE(frame_list); - mfn_list = (unsigned long *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); - if (mfn_list == NULL) - return ENOMEM; + mtx_lock(&balloon_lock); + for (entry = STAILQ_FIRST(&ballooned_pages), i = 0; + i < nr_pages; i++, entry = STAILQ_NEXT(entry, list)) { + KASSERT(entry, ("ballooned_pages list corrupt")); + page = entry->page; + frame_list[i] = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT); + } - reservation.extent_start = mfn_list; + set_xen_guest_handle(reservation.extent_start, frame_list); reservation.nr_extents = nr_pages; rc = HYPERVISOR_memory_op( - XENMEM_increase_reservation, &reservation); + XENMEM_populate_physmap, &reservation); if (rc < nr_pages) { - int ret; - /* We hit the Xen hard limit: reprobe. */ - reservation.extent_start = mfn_list; - reservation.nr_extents = rc; - ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, - &reservation); - PANIC_IF(ret != rc); - hard_limit = current_pages + rc - driver_pages; + if (rc > 0) { + int ret; + + /* We hit the Xen hard limit: reprobe. */ + reservation.nr_extents = rc; + ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, + &reservation); + KASSERT(ret == rc, ("HYPERVISOR_memory_op failed")); + } + if (rc >= 0) + bs.hard_limit = (bs.current_pages + rc - + bs.driver_pages); goto out; } for (i = 0; i < nr_pages; i++) { page = balloon_retrieve(); - PANIC_IF(page == NULL); + KASSERT(page, ("balloon_retrieve failed")); pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT); - PANIC_IF(phys_to_machine_mapping_valid(pfn)); + KASSERT((xen_feature(XENFEAT_auto_translated_physmap) || + !phys_to_machine_mapping_valid(pfn)), + ("auto translated physmap but mapping is valid")); + + set_phys_to_machine(pfn, frame_list[i]); + +#ifndef XENHVM + /* Link back into the page tables if not highmem. */ + if (pfn < max_low_pfn) { + int ret; + ret = HYPERVISOR_update_va_mapping( + (unsigned long)__va(pfn << PAGE_SHIFT), + pfn_pte_ma(frame_list[i], PAGE_KERNEL), + 0); + PASSING(ret == 0, + ("HYPERVISOR_update_va_mapping failed")); + } +#endif - /* Update P->M and M->P tables. */ - PFNTOMFN(pfn) = mfn_list[i]; - xen_machphys_update(mfn_list[i], pfn); - /* Relinquish the page back to the allocator. */ - ClearPageReserved(page); - set_page_count(page, 1); + vm_page_unwire(page, 0); vm_page_free(page); } - current_pages += nr_pages; - totalram_pages = current_pages; + bs.current_pages += nr_pages; + //totalram_pages = bs.current_pages; out: - balloon_unlock(flags); - - free((mfn_list); + mtx_unlock(&balloon_lock); return 0; } -static int +static int decrease_reservation(unsigned long nr_pages) { - unsigned long *mfn_list, pfn, i, flags; - struct page *page; - void *v; + unsigned long pfn, i; + vm_page_t page; int need_sleep = 0; int ret; struct xen_memory_reservation reservation = { @@ -209,48 +296,68 @@ decrease_reservation(unsigned long nr_pa .domid = DOMID_SELF }; - if (nr_pages > (PAGE_SIZE / sizeof(unsigned long))) - nr_pages = PAGE_SIZE / sizeof(unsigned long); - - mfn_list = (unsigned long *)malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); - if (mfn_list == NULL) - return ENOMEM; + if (nr_pages > ARRAY_SIZE(frame_list)) + nr_pages = ARRAY_SIZE(frame_list); for (i = 0; i < nr_pages; i++) { int color = 0; if ((page = vm_page_alloc(NULL, color++, - VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | - VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { + VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | + VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { nr_pages = i; need_sleep = 1; break; } + pfn = (VM_PAGE_TO_PHYS(page) >> PAGE_SHIFT); - mfn_list[i] = PFNTOMFN(pfn); + frame_list[i] = PFNTOMFN(pfn); + +#if 0 + if (!PageHighMem(page)) { + v = phys_to_virt(pfn << PAGE_SHIFT); + scrub_pages(v, 1); +#ifdef CONFIG_XEN + ret = HYPERVISOR_update_va_mapping( + (unsigned long)v, __pte_ma(0), 0); + BUG_ON(ret); +#endif + } +#endif +#ifdef CONFIG_XEN_SCRUB_PAGES + else { + v = kmap(page); + scrub_pages(v, 1); + kunmap(page); + } +#endif } - balloon_lock(flags); +#ifdef CONFIG_XEN + /* Ensure that ballooned highmem pages don't have kmaps. */ + kmap_flush_unused(); + flush_tlb_all(); +#endif + + mtx_lock(&balloon_lock); /* No more mappings: invalidate P2M and add to balloon. */ for (i = 0; i < nr_pages; i++) { - pfn = MFNTOPFN(mfn_list[i]); - PFNTOMFN(pfn) = INVALID_P2M_ENTRY; + pfn = MFNTOPFN(frame_list[i]); + set_phys_to_machine(pfn, INVALID_P2M_ENTRY); balloon_append(PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT)); } - reservation.extent_start = mfn_list; + set_xen_guest_handle(reservation.extent_start, frame_list); reservation.nr_extents = nr_pages; ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation); - PANIC_IF(ret != nr_pages); + KASSERT(ret == nr_pages, ("HYPERVISOR_memory_op failed")); - current_pages -= nr_pages; - totalram_pages = current_pages; - - balloon_unlock(flags); + bs.current_pages -= nr_pages; + //totalram_pages = bs.current_pages; - free(mfn_list, M_DEVBUF); + mtx_unlock(&balloon_lock); - return need_sleep; + return (need_sleep); } /* @@ -265,27 +372,24 @@ balloon_process(void *unused) int need_sleep = 0; long credit; + mtx_lock(&balloon_mutex); for (;;) { do { - credit = current_target() - current_pages; + credit = current_target() - bs.current_pages; if (credit > 0) need_sleep = (increase_reservation(credit) != 0); if (credit < 0) need_sleep = (decrease_reservation(-credit) != 0); -#ifndef CONFIG_PREEMPT - if (need_resched()) - schedule(); -#endif } while ((credit != 0) && !need_sleep); /* Schedule more work if there is some still to be done. */ - if (current_target() != current_pages) - timeout(balloon_alarm, NULL, ticks + HZ); + if (current_target() != bs.current_pages) + timeout(balloon_alarm, NULL, ticks + hz); - msleep(balloon_process, balloon_lock, 0, "balloon", -1); + msleep(balloon_process, &balloon_mutex, 0, "balloon", -1); } - + mtx_unlock(&balloon_mutex); } /* Resets the Xen limit, sets new target, and kicks off processing. */ @@ -293,8 +397,8 @@ static void set_new_target(unsigned long target) { /* No need for lock. Not read-modify-write updates. */ - hard_limit = ~0UL; - target_pages = target; + bs.hard_limit = ~0UL; + bs.target_pages = max(target, minimum_target()); wakeup(balloon_process); } @@ -311,8 +415,9 @@ watch_target(struct xenbus_watch *watch, unsigned long long new_target; int err; - err = xenbus_scanf(NULL, "memory", "target", "%llu", &new_target); - if (err != 1) { + err = xenbus_scanf(XBT_NIL, "memory", "target", NULL, + "%llu", &new_target); + if (err) { /* This is ok (for domain0 at least) - so just return */ return; } @@ -325,7 +430,7 @@ watch_target(struct xenbus_watch *watch, } static void -balloon_init_watcher(void *) +balloon_init_watcher(void *arg) { int err; @@ -334,48 +439,60 @@ balloon_init_watcher(void *) printf("Failed to set balloon watcher\n"); } +SYSINIT(balloon_init_watcher, SI_SUB_CONFIGURE, SI_ORDER_ANY, + balloon_init_watcher, NULL); static void -balloon_init(void *) +balloon_init(void *arg) { - unsigned long pfn; - struct page *page; +#ifndef XENHVM + vm_page_t page; +#endif - IPRINTK("Initialising balloon driver.\n"); + if (!is_running_on_xen()) + return; - if (xen_init() < 0) - return -1; - - current_pages = min(xen_start_info->nr_pages, max_pfn); - target_pages = current_pages; - balloon_low = 0; - balloon_high = 0; - driver_pages = 0UL; - hard_limit = ~0UL; - - init_timer(&balloon_timer); - balloon_timer.data = 0; - balloon_timer.function = balloon_alarm; + mtx_init(&balloon_lock, "balloon_lock", NULL, MTX_DEF); + mtx_init(&balloon_mutex, "balloon_mutex", NULL, MTX_DEF); + +#ifndef XENHVM + bs.current_pages = min(xen_start_info->nr_pages, max_pfn); +#else + bs.current_pages = physmem; +#endif + bs.target_pages = bs.current_pages; + bs.balloon_low = 0; + bs.balloon_high = 0; + bs.driver_pages = 0UL; + bs.hard_limit = ~0UL; + + kthread_create(balloon_process, NULL, NULL, 0, 0, "balloon"); +// init_timer(&balloon_timer); +// balloon_timer.data = 0; +// balloon_timer.function = balloon_alarm; +#ifndef XENHVM /* Initialise the balloon with excess memory space. */ for (pfn = xen_start_info->nr_pages; pfn < max_pfn; pfn++) { page = PHYS_TO_VM_PAGE(pfn << PAGE_SHIFT); balloon_append(page); } +#endif target_watch.callback = watch_target; - return 0; + return; } +SYSINIT(balloon_init, SI_SUB_CONFIGURE, SI_ORDER_ANY, balloon_init, NULL); + +void balloon_update_driver_allowance(long delta); void balloon_update_driver_allowance(long delta) { - unsigned long flags; - - balloon_lock(flags); - driver_pages += delta; - balloon_unlock(flags); + mtx_lock(&balloon_lock); + bs.driver_pages += delta; + mtx_unlock(&balloon_lock); } #if 0 @@ -393,17 +510,18 @@ static int dealloc_pte_fn( set_pte_at(&init_mm, addr, pte, __pte_ma(0)); set_phys_to_machine(__pa(addr) >> PAGE_SHIFT, INVALID_P2M_ENTRY); ret = HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation); - PANIC_IF(ret != 1); + KASSERT(ret == 1, ("HYPERVISOR_memory_op failed")); return 0; } #endif + +#if 0 vm_page_t balloon_alloc_empty_page_range(unsigned long nr_pages) { - unsigned long flags; vm_page_t pages; - int i; + int i, rc; unsigned long *mfn_list; struct xen_memory_reservation reservation = { .address_bits = 0, @@ -422,7 +540,9 @@ balloon_alloc_empty_page_range(unsigned PFNTOMFN(i) = INVALID_P2M_ENTRY; reservation.extent_start = mfn_list; reservation.nr_extents = nr_pages; - PANIC_IF(HYPERVISOR_memory_op(XENMEM_decrease_reservation, &reservation) != nr_pages); + rc = HYPERVISOR_memory_op(XENMEM_decrease_reservation, + &reservation); + KASSERT(rc == nr_pages, ("HYPERVISOR_memory_op failed")); } current_pages -= nr_pages; @@ -435,12 +555,11 @@ balloon_alloc_empty_page_range(unsigned void balloon_dealloc_empty_page_range(vm_page_t page, unsigned long nr_pages) { - unsigned long i, flags; + unsigned long i; for (i = 0; i < nr_pages; i++) balloon_append(page + i); wakeup(balloon_process); } - #endif From owner-svn-src-user@FreeBSD.ORG Fri Mar 6 23:22:24 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 299881065787; Fri, 6 Mar 2009 23:22:24 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id F23078FC08; Fri, 6 Mar 2009 23:22:23 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n26NMN3o031529; Fri, 6 Mar 2009 23:22:23 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n26NMN5S031528; Fri, 6 Mar 2009 23:22:23 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903062322.n26NMN5S031528@svn.freebsd.org> From: Kip Macy Date: Fri, 6 Mar 2009 23:22:23 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189458 - user/kmacy/releng_7_net_backport X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Fri, 06 Mar 2009 23:22:24 -0000 Author: kmacy Date: Fri Mar 6 23:22:23 2009 New Revision: 189458 URL: http://svn.freebsd.org/changeset/base/189458 Log: create branch for backporting networking stack Added: user/kmacy/releng_7_net_backport/ - copied from r189456, stable/7/ From owner-svn-src-user@FreeBSD.ORG Sat Mar 7 01:18:22 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 36DFC1065672; Sat, 7 Mar 2009 01:18:22 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 222C08FC1A; Sat, 7 Mar 2009 01:18:22 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n271IMpV034128; Sat, 7 Mar 2009 01:18:22 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n271IL5p034118; Sat, 7 Mar 2009 01:18:21 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903070118.n271IL5p034118@svn.freebsd.org> From: Kip Macy Date: Sat, 7 Mar 2009 01:18:21 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189470 - in user/kmacy/releng_7_net_backport/sys: amd64/include arm/include dev/bce dev/mxge i386/include ia64/include powerpc/include sparc64/include sun4v/include X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 07 Mar 2009 01:18:22 -0000 Author: kmacy Date: Sat Mar 7 01:18:21 2009 New Revision: 189470 URL: http://svn.freebsd.org/changeset/base/189470 Log: multiq step 1: - atomic_barriers: add atomic barriers to atomic.h on all architectures Modified: user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Modified: user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -32,6 +32,10 @@ #error this file needs sys/cdefs.h as a prerequisite #endif +#define mb() __asm__ __volatile__ ("mfence;": : :"memory") +#define wmb() __asm__ __volatile__ ("sfence;": : :"memory") +#define rmb() __asm__ __volatile__ ("lfence;": : :"memory") + /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. Modified: user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -45,6 +45,10 @@ #include +#define mb() +#define wmb() +#define rmb() + #ifndef I32_bit #define I32_bit (1 << 7) /* IRQ disable */ #endif Modified: user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h Sat Mar 7 01:18:21 2009 (r189470) @@ -557,6 +557,8 @@ #endif /* BCE_DEBUG */ + +#if __FreeBSD_version < 701101 #if defined(__i386__) || defined(__amd64__) #define mb() __asm volatile("mfence" ::: "memory") #define wmb() __asm volatile("sfence" ::: "memory") @@ -566,6 +568,7 @@ #define rmb() #define wmb() #endif +#endif /****************************************************************************/ /* Device identification definitions. */ Modified: user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c ============================================================================== --- user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c Sat Mar 7 01:18:21 2009 (r189470) @@ -747,9 +747,9 @@ mxge_load_firmware_helper(mxge_softc_t * mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, inflate_buffer + i, min(256U, (unsigned)(fw_len - i))); - mb(); + wmb(); dummy = *sc->sram; - mb(); + wmb(); } *limit = fw_len; @@ -782,7 +782,7 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en /* clear confirmation addr */ confirm = (volatile uint32_t *)sc->cmd; *confirm = 0; - mb(); + wmb(); /* send an rdma command to the PCIe engine, and wait for the response in the confirmation address. The firmware should @@ -804,9 +804,9 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); mxge_pio_copy(submit, buf, 64); - mb(); + wmb(); DELAY(1000); - mb(); + wmb(); i = 0; while (*confirm != 0xffffffff && i < 20) { DELAY(1000); @@ -844,7 +844,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t buf->response_addr.high = htobe32(dma_high); mtx_lock(&sc->cmd_mtx); response->result = 0xffffffff; - mb(); + wmb(); mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); /* wait up to 20ms */ @@ -852,7 +852,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t for (sleep_total = 0; sleep_total < 20; sleep_total++) { bus_dmamap_sync(sc->cmd_dma.dmat, sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); - mb(); + wmb(); switch (be32toh(response->result)) { case 0: data->data0 = be32toh(response->data); @@ -980,7 +980,7 @@ mxge_load_firmware(mxge_softc_t *sc, int /* clear confirmation addr */ confirm = (volatile uint32_t *)sc->cmd; *confirm = 0; - mb(); + wmb(); /* send a reload command to the bootstrap MCP, and wait for the response in the confirmation address. The firmware should write a -1 there to indicate it is alive and well @@ -1005,9 +1005,9 @@ mxge_load_firmware(mxge_softc_t *sc, int submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); mxge_pio_copy(submit, buf, 64); - mb(); + wmb(); DELAY(1000); - mb(); + wmb(); i = 0; while (*confirm != 0xffffffff && i < 20) { DELAY(1000*10); @@ -1668,7 +1668,7 @@ mxge_submit_req_backwards(mxge_tx_ring_t idx = (starting_slot + cnt) & tx->mask; mxge_pio_copy(&tx->lanai[idx], &src[cnt], sizeof(*src)); - mb(); + wmb(); } } @@ -1694,14 +1694,14 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ last_flags = src->flags; src->flags = 0; - mb(); + wmb(); dst = dstp = &tx->lanai[idx]; srcp = src; if ((idx + cnt) < tx->mask) { for (i = 0; i < (cnt - 1); i += 2) { mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); - mb(); /* force write every 32 bytes */ + wmb(); /* force write every 32 bytes */ srcp += 2; dstp += 2; } @@ -1714,7 +1714,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ if (i < cnt) { /* submit the first request */ mxge_pio_copy(dstp, srcp, sizeof(*src)); - mb(); /* barrier before setting valid flag */ + wmb(); /* barrier before setting valid flag */ } /* re-write the last 32-bits with the valid flags */ @@ -1725,7 +1725,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ dst_ints+=3; *dst_ints = *src_ints; tx->req += cnt; - mb(); + wmb(); } #if IFCAP_TSO4 @@ -2144,12 +2144,12 @@ mxge_submit_8rx(volatile mcp_kreq_ether_ low = src->addr_low; src->addr_low = 0xffffffff; mxge_pio_copy(dst, src, 4 * sizeof (*src)); - mb(); + wmb(); mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); - mb(); + wmb(); src->addr_low = low; dst->addr_low = low; - mb(); + wmb(); } static int @@ -2736,7 +2736,7 @@ mxge_intr(void *arg) send_done_count = be32toh(stats->send_done_count); } if (sc->legacy_irq && mxge_deassert_wait) - mb(); + wmb(); } while (*((volatile uint8_t *) &stats->valid)); if (__predict_false(stats->stats_updated)) { @@ -3385,7 +3385,7 @@ mxge_close(mxge_softc_t *sc) callout_stop(&sc->co_hdl); sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; old_down_cnt = sc->down_cnt; - mb(); + wmb(); err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); if (err) { device_printf(sc->dev, "Couldn't bring down link\n"); @@ -3394,7 +3394,7 @@ mxge_close(mxge_softc_t *sc) /* wait for down irq */ DELAY(10 * sc->intr_coal_delay); } - mb(); + wmb(); if (old_down_cnt == sc->down_cnt) { device_printf(sc->dev, "never got down irq\n"); } Modified: user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h Sat Mar 7 01:18:21 2009 (r189470) @@ -282,6 +282,8 @@ struct mxge_media_type /* implement our own memory barriers, since bus_space_barrier cannot handle write-combining regions */ +#if __FreeBSD_version < 701101 + #if defined (__GNUC__) #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__ || #cpu(x86_64) || defined __x86_64__ #define mb() __asm__ __volatile__ ("sfence;": : :"memory") @@ -296,6 +298,8 @@ struct mxge_media_type #error "unknown compiler" #endif +#endif + static inline void mxge_pio_copy(volatile void *to_v, void *from_v, size_t size) { Modified: user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -32,6 +32,47 @@ #error this file needs sys/cdefs.h as a prerequisite #endif +#if defined(_KERNEL) +#include +#include +#define mb() __asm __volatile( \ + "testl %0,cpu_feature \n\ + je 2f \n\ + mfence \n\ +1: \n\ + .section .text.offpath \n\ +2: lock \n\ + addl $0,cpu_feature \n\ + jmp 1b \n\ + .text \n\ +" \ + : : "i"(CPUID_SSE2) : "memory") +#define wmb() __asm __volatile( \ + "testl %0,cpu_feature \n\ + je 2f \n\ + sfence \n\ +1: \n\ + .section .text.offpath \n\ +2: lock \n\ + addl $0,cpu_feature \n\ + jmp 1b \n\ + .text \n\ +" \ + : : "i"(CPUID_XMM) : "memory") +#define rmb() __asm __volatile( \ + "testl %0,cpu_feature \n\ + je 2f \n\ + lfence \n\ +1: \n\ + .section .text.offpath \n\ +2: lock \n\ + addl $0,cpu_feature \n\ + jmp 1b \n\ + .text \n\ +" \ + : : "i"(CPUID_SSE2) : "memory") +#endif + /* * Various simple operations on memory, each of which is atomic in the * presence of interrupts and multiple processors. Modified: user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -29,6 +29,10 @@ #ifndef _MACHINE_ATOMIC_H_ #define _MACHINE_ATOMIC_H_ +#define mb() +#define wmb() +#define rmb() + /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and SMP safe. Modified: user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -39,6 +39,10 @@ #define __ATOMIC_BARRIER \ __asm __volatile("sync" : : : "memory") +#define mb() __ATOMIC_BARRIER +#define wmb() mb() +#define rmb() mb() + /* * atomic_add(p, v) * { *p += v; } Modified: user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -40,6 +40,10 @@ #define __ASI_ATOMIC ASI_P #endif +#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") +#define wmb() mb() +#define rmb() mb() + /* * Various simple arithmetic on memory which is atomic in the presence * of interrupts and multiple processors. See atomic(9) for details. Modified: user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h ============================================================================== --- user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Sat Mar 7 01:17:13 2009 (r189469) +++ user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Sat Mar 7 01:18:21 2009 (r189470) @@ -33,6 +33,10 @@ #include +#define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") +#define wmb() mb() +#define rmb() mb() + /* Userland needs different ASI's. */ #ifdef _KERNEL #define __ASI_ATOMIC ASI_N From owner-svn-src-user@FreeBSD.ORG Sat Mar 7 05:39:56 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 946D9106566C; Sat, 7 Mar 2009 05:39:56 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 7EDFC8FC0C; Sat, 7 Mar 2009 05:39:56 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n275duug040116; Sat, 7 Mar 2009 05:39:56 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n275du4R040104; Sat, 7 Mar 2009 05:39:56 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200903070539.n275du4R040104@svn.freebsd.org> From: Kip Macy Date: Sat, 7 Mar 2009 05:39:56 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r189485 - in user/kmacy/releng_7_net_backport: sbin/route sys/amd64/conf sys/conf sys/kern sys/net sys/netinet sys/netinet6 sys/sys usr.bin/netstat usr.sbin/ndp usr.sbin/ppp usr.sbin/ro... X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 07 Mar 2009 05:39:56 -0000 Author: kmacy Date: Sat Mar 7 05:39:55 2009 New Revision: 189485 URL: http://svn.freebsd.org/changeset/base/189485 Log: Backport HEAD network stack as of r189450 Added: user/kmacy/releng_7_net_backport/sys/kern/kern_rmlock.c user/kmacy/releng_7_net_backport/sys/kern/subr_bufring.c user/kmacy/releng_7_net_backport/sys/net/bpf_buffer.c user/kmacy/releng_7_net_backport/sys/net/bpf_buffer.h user/kmacy/releng_7_net_backport/sys/net/bpf_zerocopy.c user/kmacy/releng_7_net_backport/sys/net/bpf_zerocopy.h user/kmacy/releng_7_net_backport/sys/net/if_llatbl.c user/kmacy/releng_7_net_backport/sys/net/if_llatbl.h user/kmacy/releng_7_net_backport/sys/net/radix_mpath.c user/kmacy/releng_7_net_backport/sys/net/radix_mpath.h user/kmacy/releng_7_net_backport/sys/net/vnet.h user/kmacy/releng_7_net_backport/sys/netinet/accf_dns.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_hostcache.h user/kmacy/releng_7_net_backport/sys/netinet/vinet.h user/kmacy/releng_7_net_backport/sys/netinet6/vinet6.h user/kmacy/releng_7_net_backport/sys/sys/_rmlock.h user/kmacy/releng_7_net_backport/sys/sys/buf_ring.h user/kmacy/releng_7_net_backport/sys/sys/rmlock.h user/kmacy/releng_7_net_backport/sys/sys/vimage.h Modified: user/kmacy/releng_7_net_backport/sbin/route/route.c user/kmacy/releng_7_net_backport/sys/amd64/conf/GENERIC user/kmacy/releng_7_net_backport/sys/conf/files user/kmacy/releng_7_net_backport/sys/conf/options user/kmacy/releng_7_net_backport/sys/kern/kern_mib.c user/kmacy/releng_7_net_backport/sys/kern/kern_uuid.c user/kmacy/releng_7_net_backport/sys/kern/subr_pcpu.c user/kmacy/releng_7_net_backport/sys/kern/vfs_export.c user/kmacy/releng_7_net_backport/sys/net/bpf.c user/kmacy/releng_7_net_backport/sys/net/bpf.h user/kmacy/releng_7_net_backport/sys/net/bpf_filter.c user/kmacy/releng_7_net_backport/sys/net/bpfdesc.h user/kmacy/releng_7_net_backport/sys/net/bridgestp.c user/kmacy/releng_7_net_backport/sys/net/bridgestp.h user/kmacy/releng_7_net_backport/sys/net/bsd_comp.c user/kmacy/releng_7_net_backport/sys/net/ethernet.h user/kmacy/releng_7_net_backport/sys/net/ieee8023ad_lacp.c user/kmacy/releng_7_net_backport/sys/net/if.c user/kmacy/releng_7_net_backport/sys/net/if.h user/kmacy/releng_7_net_backport/sys/net/if_arcsubr.c user/kmacy/releng_7_net_backport/sys/net/if_atmsubr.c user/kmacy/releng_7_net_backport/sys/net/if_bridge.c user/kmacy/releng_7_net_backport/sys/net/if_clone.c user/kmacy/releng_7_net_backport/sys/net/if_clone.h user/kmacy/releng_7_net_backport/sys/net/if_disc.c user/kmacy/releng_7_net_backport/sys/net/if_ef.c user/kmacy/releng_7_net_backport/sys/net/if_enc.c user/kmacy/releng_7_net_backport/sys/net/if_ethersubr.c user/kmacy/releng_7_net_backport/sys/net/if_faith.c user/kmacy/releng_7_net_backport/sys/net/if_fddisubr.c user/kmacy/releng_7_net_backport/sys/net/if_fwsubr.c user/kmacy/releng_7_net_backport/sys/net/if_gif.c user/kmacy/releng_7_net_backport/sys/net/if_gif.h user/kmacy/releng_7_net_backport/sys/net/if_gre.c user/kmacy/releng_7_net_backport/sys/net/if_iso88025subr.c user/kmacy/releng_7_net_backport/sys/net/if_lagg.c user/kmacy/releng_7_net_backport/sys/net/if_loop.c user/kmacy/releng_7_net_backport/sys/net/if_media.h user/kmacy/releng_7_net_backport/sys/net/if_mib.c user/kmacy/releng_7_net_backport/sys/net/if_ppp.c user/kmacy/releng_7_net_backport/sys/net/if_sl.c user/kmacy/releng_7_net_backport/sys/net/if_spppsubr.c user/kmacy/releng_7_net_backport/sys/net/if_stf.c user/kmacy/releng_7_net_backport/sys/net/if_tap.c user/kmacy/releng_7_net_backport/sys/net/if_tun.c user/kmacy/releng_7_net_backport/sys/net/if_var.h user/kmacy/releng_7_net_backport/sys/net/if_vlan.c user/kmacy/releng_7_net_backport/sys/net/netisr.c user/kmacy/releng_7_net_backport/sys/net/netisr.h user/kmacy/releng_7_net_backport/sys/net/pfil.c user/kmacy/releng_7_net_backport/sys/net/pfil.h user/kmacy/releng_7_net_backport/sys/net/ppp_deflate.c user/kmacy/releng_7_net_backport/sys/net/radix.c user/kmacy/releng_7_net_backport/sys/net/radix.h user/kmacy/releng_7_net_backport/sys/net/raw_cb.c user/kmacy/releng_7_net_backport/sys/net/raw_cb.h user/kmacy/releng_7_net_backport/sys/net/raw_usrreq.c user/kmacy/releng_7_net_backport/sys/net/route.c user/kmacy/releng_7_net_backport/sys/net/route.h user/kmacy/releng_7_net_backport/sys/net/rtsock.c user/kmacy/releng_7_net_backport/sys/netinet/icmp6.h user/kmacy/releng_7_net_backport/sys/netinet/icmp_var.h user/kmacy/releng_7_net_backport/sys/netinet/if_atm.c user/kmacy/releng_7_net_backport/sys/netinet/if_ether.c user/kmacy/releng_7_net_backport/sys/netinet/if_ether.h user/kmacy/releng_7_net_backport/sys/netinet/igmp.c user/kmacy/releng_7_net_backport/sys/netinet/igmp.h user/kmacy/releng_7_net_backport/sys/netinet/in.c user/kmacy/releng_7_net_backport/sys/netinet/in.h user/kmacy/releng_7_net_backport/sys/netinet/in_gif.c user/kmacy/releng_7_net_backport/sys/netinet/in_mcast.c user/kmacy/releng_7_net_backport/sys/netinet/in_pcb.c user/kmacy/releng_7_net_backport/sys/netinet/in_pcb.h user/kmacy/releng_7_net_backport/sys/netinet/in_proto.c user/kmacy/releng_7_net_backport/sys/netinet/in_rmx.c user/kmacy/releng_7_net_backport/sys/netinet/in_var.h user/kmacy/releng_7_net_backport/sys/netinet/ip.h user/kmacy/releng_7_net_backport/sys/netinet/ip6.h user/kmacy/releng_7_net_backport/sys/netinet/ip_carp.c user/kmacy/releng_7_net_backport/sys/netinet/ip_divert.c user/kmacy/releng_7_net_backport/sys/netinet/ip_dummynet.c user/kmacy/releng_7_net_backport/sys/netinet/ip_dummynet.h user/kmacy/releng_7_net_backport/sys/netinet/ip_fastfwd.c user/kmacy/releng_7_net_backport/sys/netinet/ip_fw.h user/kmacy/releng_7_net_backport/sys/netinet/ip_fw2.c user/kmacy/releng_7_net_backport/sys/netinet/ip_fw_nat.c user/kmacy/releng_7_net_backport/sys/netinet/ip_fw_pfil.c user/kmacy/releng_7_net_backport/sys/netinet/ip_gre.c user/kmacy/releng_7_net_backport/sys/netinet/ip_icmp.c user/kmacy/releng_7_net_backport/sys/netinet/ip_icmp.h user/kmacy/releng_7_net_backport/sys/netinet/ip_id.c user/kmacy/releng_7_net_backport/sys/netinet/ip_input.c user/kmacy/releng_7_net_backport/sys/netinet/ip_ipsec.c user/kmacy/releng_7_net_backport/sys/netinet/ip_mroute.c user/kmacy/releng_7_net_backport/sys/netinet/ip_options.c user/kmacy/releng_7_net_backport/sys/netinet/ip_options.h user/kmacy/releng_7_net_backport/sys/netinet/ip_output.c user/kmacy/releng_7_net_backport/sys/netinet/ip_var.h user/kmacy/releng_7_net_backport/sys/netinet/raw_ip.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_auth.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_bsd_addr.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_constants.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_crc32.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_crc32.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_header.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_indata.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_input.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_os_bsd.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_output.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_output.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_pcb.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_pcb.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_structs.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_sysctl.c user/kmacy/releng_7_net_backport/sys/netinet/sctp_uio.h user/kmacy/releng_7_net_backport/sys/netinet/sctp_usrreq.c user/kmacy/releng_7_net_backport/sys/netinet/sctputil.c user/kmacy/releng_7_net_backport/sys/netinet/sctputil.h user/kmacy/releng_7_net_backport/sys/netinet/tcp_hostcache.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_input.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_offload.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_offload.h user/kmacy/releng_7_net_backport/sys/netinet/tcp_output.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_reass.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_sack.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_subr.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_syncache.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_syncache.h user/kmacy/releng_7_net_backport/sys/netinet/tcp_timer.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_timewait.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_usrreq.c user/kmacy/releng_7_net_backport/sys/netinet/tcp_var.h user/kmacy/releng_7_net_backport/sys/netinet/toedev.h user/kmacy/releng_7_net_backport/sys/netinet/udp_usrreq.c user/kmacy/releng_7_net_backport/sys/netinet/udp_var.h user/kmacy/releng_7_net_backport/sys/netinet6/dest6.c user/kmacy/releng_7_net_backport/sys/netinet6/frag6.c user/kmacy/releng_7_net_backport/sys/netinet6/icmp6.c user/kmacy/releng_7_net_backport/sys/netinet6/in6.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_gif.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_ifattach.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_pcb.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_pcb.h user/kmacy/releng_7_net_backport/sys/netinet6/in6_proto.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_rmx.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_src.c user/kmacy/releng_7_net_backport/sys/netinet6/in6_var.h user/kmacy/releng_7_net_backport/sys/netinet6/ip6_forward.c user/kmacy/releng_7_net_backport/sys/netinet6/ip6_input.c user/kmacy/releng_7_net_backport/sys/netinet6/ip6_ipsec.c user/kmacy/releng_7_net_backport/sys/netinet6/ip6_mroute.c user/kmacy/releng_7_net_backport/sys/netinet6/ip6_output.c user/kmacy/releng_7_net_backport/sys/netinet6/ip6_var.h user/kmacy/releng_7_net_backport/sys/netinet6/ip6protosw.h user/kmacy/releng_7_net_backport/sys/netinet6/mld6.c user/kmacy/releng_7_net_backport/sys/netinet6/nd6.c user/kmacy/releng_7_net_backport/sys/netinet6/nd6.h user/kmacy/releng_7_net_backport/sys/netinet6/nd6_nbr.c user/kmacy/releng_7_net_backport/sys/netinet6/nd6_rtr.c user/kmacy/releng_7_net_backport/sys/netinet6/raw_ip6.c user/kmacy/releng_7_net_backport/sys/netinet6/raw_ip6.h user/kmacy/releng_7_net_backport/sys/netinet6/route6.c user/kmacy/releng_7_net_backport/sys/netinet6/scope6.c user/kmacy/releng_7_net_backport/sys/netinet6/sctp6_usrreq.c user/kmacy/releng_7_net_backport/sys/netinet6/tcp6_var.h user/kmacy/releng_7_net_backport/sys/netinet6/udp6_usrreq.c user/kmacy/releng_7_net_backport/sys/sys/conf.h user/kmacy/releng_7_net_backport/sys/sys/kernel.h user/kmacy/releng_7_net_backport/sys/sys/mbuf.h user/kmacy/releng_7_net_backport/sys/sys/pcpu.h user/kmacy/releng_7_net_backport/sys/sys/sysctl.h user/kmacy/releng_7_net_backport/usr.bin/netstat/inet6.c user/kmacy/releng_7_net_backport/usr.bin/netstat/route.c user/kmacy/releng_7_net_backport/usr.sbin/ndp/ndp.c user/kmacy/releng_7_net_backport/usr.sbin/ppp/route.c user/kmacy/releng_7_net_backport/usr.sbin/route6d/route6d.c Modified: user/kmacy/releng_7_net_backport/sbin/route/route.c ============================================================================== --- user/kmacy/releng_7_net_backport/sbin/route/route.c Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sbin/route/route.c Sat Mar 7 05:39:55 2009 (r189485) @@ -605,9 +605,6 @@ newroute(argc, argv) case K_NOSTATIC: flags &= ~RTF_STATIC; break; - case K_LLINFO: - flags |= RTF_LLINFO; - break; case K_LOCK: locking = 1; break; @@ -632,9 +629,6 @@ newroute(argc, argv) case K_PROXY: proxy = 1; break; - case K_CLONING: - flags |= RTF_CLONING; - break; case K_XRESOLVE: flags |= RTF_XRESOLVE; break; Modified: user/kmacy/releng_7_net_backport/sys/amd64/conf/GENERIC ============================================================================== --- user/kmacy/releng_7_net_backport/sys/amd64/conf/GENERIC Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/amd64/conf/GENERIC Sat Mar 7 05:39:55 2009 (r189485) @@ -30,7 +30,7 @@ options SCHED_ULE # ULE scheduler options PREEMPTION # Enable kernel thread preemption options INET # InterNETworking options INET6 # IPv6 communications protocols -options SCTP # Stream Control Transmission Protocol +#options SCTP # Stream Control Transmission Protocol options FFS # Berkeley Fast Filesystem options SOFTUPDATES # Enable FFS soft updates support options UFS_ACL # Support for access control lists @@ -306,3 +306,6 @@ device fwe # Ethernet over FireWire (n device fwip # IP over FireWire (RFC 2734,3146) device dcons # Dumb console driver device dcons_crom # Configuration ROM for dcons + +options VIMAGE_GLOBALS + Modified: user/kmacy/releng_7_net_backport/sys/conf/files ============================================================================== --- user/kmacy/releng_7_net_backport/sys/conf/files Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/conf/files Sat Mar 7 05:39:55 2009 (r189485) @@ -1489,6 +1489,7 @@ kern/subr_acl_posix1e.c standard kern/subr_autoconf.c standard kern/subr_blist.c standard kern/subr_bus.c standard +kern/subr_bufring.c standard kern/subr_clock.c standard kern/subr_devstat.c standard kern/subr_disk.c standard @@ -1513,6 +1514,7 @@ kern/subr_rman.c standard kern/subr_rtc.c optional genclock kern/subr_sbuf.c standard kern/subr_scanf.c standard +kern/kern_rmlock.c standard kern/kern_sdt.c optional kdtrace_hooks kern/subr_sleepqueue.c standard kern/subr_smp.c standard @@ -1611,8 +1613,10 @@ libkern/strtoul.c standard libkern/strtouq.c standard libkern/strvalid.c standard net/bpf.c standard +net/bpf_buffer.c optional bpf net/bpf_jitter.c optional bpf_jitter net/bpf_filter.c optional bpf | netgraph_bpf +net/bpf_zerocopy.c optional bpf net/bridgestp.c optional bridge | if_bridge net/bsd_comp.c optional ppp_bsdcomp net/ieee8023ad_lacp.c optional lagg @@ -1635,6 +1639,7 @@ net/if_gre.c optional gre net/if_iso88025subr.c optional token net/if_lagg.c optional lagg net/if_loop.c optional loop +net/if_llatbl.c standard net/if_media.c standard net/if_mib.c standard net/if_ppp.c optional ppp @@ -1652,6 +1657,7 @@ net/ppp_deflate.c optional ppp_deflate net/ppp_tty.c optional ppp net/pfil.c optional ether | inet net/radix.c standard +net/radix_mpath.c standard net/raw_cb.c standard net/raw_usrreq.c standard net/route.c standard Modified: user/kmacy/releng_7_net_backport/sys/conf/options ============================================================================== --- user/kmacy/releng_7_net_backport/sys/conf/options Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/conf/options Sat Mar 7 05:39:55 2009 (r189485) @@ -373,6 +373,7 @@ IPSEC opt_ipsec.h IPSEC_DEBUG opt_ipsec.h IPSEC_FILTERTUNNEL opt_ipsec.h IPDIVERT +MBUF_PROFILING DUMMYNET opt_ipdn.h IPFILTER opt_ipfilter.h IPFILTER_LOG opt_ipfilter.h @@ -395,6 +396,7 @@ NETATALK opt_atalk.h PPP_BSDCOMP opt_ppp.h PPP_DEFLATE opt_ppp.h PPP_FILTER opt_ppp.h +RADIX_MPATH opt_mpath.h SLIP_IFF_OPTS opt_slip.h TCPDEBUG TCP_SIGNATURE opt_inet.h @@ -754,3 +756,7 @@ XFS # Interrupt filtering INTR_FILTER opt_global.h + +# Virtualize the network stack +VIMAGE opt_global.h +VIMAGE_GLOBALS opt_global.h Modified: user/kmacy/releng_7_net_backport/sys/kern/kern_mib.c ============================================================================== --- user/kmacy/releng_7_net_backport/sys/kern/kern_mib.c Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/kern/kern_mib.c Sat Mar 7 05:39:55 2009 (r189485) @@ -208,6 +208,9 @@ SYSCTL_STRING(_hw, HW_MACHINE_ARCH, mach char hostname[MAXHOSTNAMELEN]; +struct mtx hostname_mtx; +MTX_SYSINIT(hostname_mtx, &hostname_mtx, "hostname", MTX_DEF); + static int sysctl_hostname(SYSCTL_HANDLER_ARGS) { Added: user/kmacy/releng_7_net_backport/sys/kern/kern_rmlock.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/releng_7_net_backport/sys/kern/kern_rmlock.c Sat Mar 7 05:39:55 2009 (r189485) @@ -0,0 +1,522 @@ +/*- + * Copyright (c) 2007 Stephan Uphoff + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Machine independent bits of reader/writer lock implementation. + */ + +#include +__FBSDID("$FreeBSD: head/sys/kern/kern_rmlock.c 182914 2008-09-10 19:13:30Z jhb $"); + +#include "opt_ddb.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef DDB +#include +#endif + +#undef WITNESS_CHECKORDER +#define WITNESS_CHECKORDER(a, b, c, d, e) + +#define RMPF_ONQUEUE 1 +#define RMPF_SIGNAL 2 + +/* + * To support usage of rmlock in CVs and msleep + * yet another list for the priority tracker + * would be needed. + * Using this lock for cv and msleep also does + * not seem very useful + */ + +static __inline void compiler_memory_barrier(void) { + __asm __volatile("":::"memory"); +} +#if 0 +static void assert_rm(struct lock_object *lock, int what); +#endif +static void lock_rm(struct lock_object *lock, int how); +static int unlock_rm(struct lock_object *lock); + +struct lock_class lock_class_rm = { + .lc_name = "rm", + .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, +#if 0 + .lc_assert = assert_rm, +#endif +#if 0 +#ifdef DDB + .lc_ddb_show = db_show_rwlock, +#endif +#endif + .lc_lock = lock_rm, + .lc_unlock = unlock_rm, +}; + +#if 0 +static void +assert_rm(struct lock_object *lock, int what) +{ + + panic("assert_rm called"); +} +#endif +static void +lock_rm(struct lock_object *lock, int how) { + panic("lock_rm called"); +} + +static int +unlock_rm(struct lock_object *lock) { + panic("unlock_rm called"); +} + +static struct mtx rm_spinlock; + +MTX_SYSINIT(rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN); + +/* + * Add or remove tracker from per cpu list. + * The per cpu list can be traversed at any time in forward + * direction from an interrupt on the *local* cpu. + */ + +static void inline +rm_tracker_add(struct pcpu *pc, struct rm_priotracker* tracker) { + struct rm_queue* next; + /* Initialize all tracker pointers */ + tracker->rmp_cpuQueue.rmq_prev = &pc->pc_rm_queue; + next = pc->pc_rm_queue.rmq_next; + tracker->rmp_cpuQueue.rmq_next = next; + /* rmq_prev is not used during froward traversal */ + next->rmq_prev = &tracker->rmp_cpuQueue; + /* Update pointer to first element */ + pc->pc_rm_queue.rmq_next = &tracker->rmp_cpuQueue; +} + + +static void inline +rm_tracker_remove(struct pcpu *pc, struct rm_priotracker* tracker) { + struct rm_queue *next, *prev; + next = tracker->rmp_cpuQueue.rmq_next; + prev = tracker->rmp_cpuQueue.rmq_prev; + /* Not used during forward traversal */ + next->rmq_prev = prev; + /* Remove from list */ + prev->rmq_next = next; +} + + + + +static void rm_cleanIPI(void *arg) { + struct pcpu *pc; + struct rmlock* rm = arg; + struct rm_priotracker *tracker; + struct rm_queue* queue; + pc = pcpu_find(curcpu); + + for(queue = pc->pc_rm_queue.rmq_next; + queue != &pc->pc_rm_queue; + queue = queue->rmq_next) { + tracker = (struct rm_priotracker *) queue; + if(tracker->rmp_rmlock == rm && tracker->rmp_flags == 0 ) { + tracker->rmp_flags = RMPF_ONQUEUE; + mtx_lock_spin(&rm_spinlock); + LIST_INSERT_HEAD(&rm->rm_activeReaders,tracker, + rmp_qentry); + mtx_unlock_spin(&rm_spinlock); + } + } + return; +} + + + +void +rm_init(struct rmlock *rm, const char *name, int opts) +{ + rm->rm_noreadtoken = 1; + LIST_INIT(&rm->rm_activeReaders); + mtx_init(&rm->rm_lock, name, "RM_MTX",MTX_NOWITNESS); + lock_init(&rm->lock_object, &lock_class_rm, name, NULL, (opts & LO_RECURSABLE)| LO_WITNESS); + +} + +void +rm_destroy(struct rmlock *rm) +{ + mtx_destroy(&rm->rm_lock); + lock_destroy(&rm->lock_object); +} + +int +rm_wowned(struct rmlock *rm) +{ + + return (mtx_owned(&rm->rm_lock)); +} + +void +rm_sysinit(void *arg) +{ + struct rm_args *args = arg; + rm_init(args->ra_rm, args->ra_desc, args->ra_opts); +} + + +static void +_rm_rlock_hard(struct rmlock *rm, struct rm_priotracker* tracker) +{ + struct pcpu *pc; + struct rm_queue *queue; + struct rm_priotracker* atracker; + + critical_enter(); + pc = pcpu_find(curcpu); + + /* Check if we just need to do a proper critical_exit */ + if (0 == rm->rm_noreadtoken) { + critical_exit(); + return; + } + + /* Remove our tracker from the per cpu list */ + rm_tracker_remove(pc,tracker); + + /* Check to see if the IPI granted us the lock after all */ + if(tracker->rmp_flags) { + /* Just add back tracker - we hold the lock */ + rm_tracker_add(pc,tracker); + critical_exit(); + return; + } + + + + /* + * We allow readers to aquire a lock even if a writer + * is blocked if the lock is recursive and the reader + * already holds the lock + */ + + if ((rm->lock_object.lo_flags & LO_RECURSABLE) != 0) { + /* + * Just grand the lock if this thread already have a tracker + * for this lock on the per cpu queue + */ + + for(queue = pc->pc_rm_queue.rmq_next; + queue != &pc->pc_rm_queue; + queue = queue->rmq_next) { + atracker = (struct rm_priotracker *) queue; + if (( atracker->rmp_rmlock == rm) && + ( atracker->rmp_thread == tracker->rmp_thread )) { + mtx_lock_spin(&rm_spinlock); + LIST_INSERT_HEAD(&rm->rm_activeReaders,tracker, + rmp_qentry); + tracker->rmp_flags = RMPF_ONQUEUE; + mtx_unlock_spin(&rm_spinlock); + rm_tracker_add(pc,tracker); + critical_exit(); + return; + } + } + } + + + sched_unpin(); + critical_exit(); + + mtx_lock(&rm->rm_lock); + rm->rm_noreadtoken = 0; + critical_enter(); + + pc = pcpu_find(curcpu); + rm_tracker_add(pc,tracker); + sched_pin(); + critical_exit(); + + mtx_unlock(&rm->rm_lock); + return; +} + +void +_rm_rlock(struct rmlock *rm, struct rm_priotracker* tracker) +{ + struct thread *td = curthread; + struct pcpu *pc; + + tracker->rmp_flags = 0; + tracker->rmp_thread = td; + tracker->rmp_rmlock = rm; + + td->td_critnest++; /* critical_enter(); */ + + compiler_memory_barrier(); + + pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ + + rm_tracker_add(pc,tracker); + + td->td_pinned++; /* sched_pin(); */ + + compiler_memory_barrier(); + + td->td_critnest--; + + /* + * Fast path to combine two common conditions + * into a single conditional jump + */ + + if (0 == (td->td_owepreempt | rm->rm_noreadtoken)) { + return; + } + + /* We do not have a read token and need to acquire one */ + _rm_rlock_hard(rm,tracker); +} + + +static void +_rm_unlock_hard(struct thread *td,struct rm_priotracker* tracker) +{ + + if (td->td_owepreempt) { + td->td_critnest++; + critical_exit(); + } + + if (!tracker->rmp_flags) { + return; + } + + + mtx_lock_spin(&rm_spinlock); + LIST_REMOVE(tracker,rmp_qentry); + + if (tracker->rmp_flags & RMPF_SIGNAL) { + struct rmlock *rm; + struct turnstile* ts; + + rm = tracker->rmp_rmlock; + + turnstile_chain_lock(&rm->lock_object); + mtx_unlock_spin(&rm_spinlock); + + ts = turnstile_lookup(&rm->lock_object); + + turnstile_signal(ts, TS_EXCLUSIVE_QUEUE); + turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); + turnstile_chain_unlock(&rm->lock_object); + + + } else + mtx_unlock_spin(&rm_spinlock); + +} + +void +_rm_runlock(struct rmlock *rm, struct rm_priotracker* tracker) +{ + struct pcpu *pc; + struct thread *td = tracker->rmp_thread; + + td->td_critnest++; /* critical_enter(); */ + pc = cpuid_to_pcpu[td->td_oncpu]; /* pcpu_find(td->td_oncpu); */ + rm_tracker_remove(pc,tracker); + td->td_critnest--; + td->td_pinned--; /* sched_unpin(); */ + + if (0 == (td->td_owepreempt | tracker->rmp_flags)) + return; + + + _rm_unlock_hard(td,tracker); + + +} + + +void +_rm_wlock(struct rmlock *rm) +{ + struct rm_priotracker *prio; + struct turnstile *ts; + + mtx_lock(&rm->rm_lock); + + if (rm->rm_noreadtoken == 0) { + /* Get all read tokens back */ + + rm->rm_noreadtoken = 1; + + /* + * Assumes rm->rm_noreadtoken update is visible + * on other CPUs before rm_cleanIPI is called + */ +#ifdef SMP + smp_rendezvous(smp_no_rendevous_barrier, + rm_cleanIPI, + smp_no_rendevous_barrier + ,rm); + +#else + rm_cleanIPI(rm); +#endif + + + mtx_lock_spin(&rm_spinlock); + + while((prio = LIST_FIRST(&rm->rm_activeReaders)) != NULL) { + ts = turnstile_trywait(&rm->lock_object); + prio->rmp_flags = RMPF_ONQUEUE | RMPF_SIGNAL; + mtx_unlock_spin(&rm_spinlock); + turnstile_wait(ts,prio->rmp_thread, + TS_EXCLUSIVE_QUEUE); + mtx_lock_spin(&rm_spinlock); + + } + + mtx_unlock_spin(&rm_spinlock); + } + +} + + +void +_rm_wunlock(struct rmlock *rm) +{ + mtx_unlock(&rm->rm_lock); +} + + +#ifdef LOCK_DEBUG + +void _rm_wlock_debug(struct rmlock *rm, const char *file, int line) +{ + + WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, + file, line, NULL); + + _rm_wlock(rm); + + LOCK_LOG_LOCK("RMWLOCK", &rm->lock_object, 0, 0, file, line); + + WITNESS_LOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); + + curthread->td_locks++; + +} + +void _rm_wunlock_debug(struct rmlock *rm, const char *file, int line) +{ + curthread->td_locks--; + WITNESS_UNLOCK(&rm->lock_object, LOP_EXCLUSIVE, file, line); + LOCK_LOG_LOCK("RMWUNLOCK", &rm->lock_object, 0, 0, file, line); + _rm_wunlock(rm); +} + + +void +_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, + const char *file, int line) +{ + + + WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER , file, line, NULL); + + _rm_rlock(rm, tracker); + + LOCK_LOG_LOCK("RMRLOCK", &rm->lock_object, 0, 0, file, line); + + WITNESS_LOCK(&rm->lock_object, 0 , file, line); + + curthread->td_locks++; +} + +void +_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, + const char *file, int line) { + curthread->td_locks--; + WITNESS_UNLOCK(&rm->lock_object, 0 , file, line); + LOCK_LOG_LOCK("RMRUNLOCK", &rm->lock_object, 0, 0, file, line); + _rm_runlock(rm, tracker); +} + + + + +#else +/* + * Just strip out file and line arguments if no lock debugging is enabled + * in the kernel - we are called from a kernel module. +*/ + + +void _rm_wlock_debug(struct rmlock *rm, const char *file, int line) +{ + _rm_wlock(rm); +} + +void _rm_wunlock_debug(struct rmlock *rm, const char *file, int line) +{ + _rm_wunlock(rm); +} + +void +_rm_rlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, + const char *file, int line) +{ + _rm_rlock(rm, tracker); +} + +void +_rm_runlock_debug(struct rmlock *rm, struct rm_priotracker *tracker, + const char *file, int line) { + _rm_runlock(rm, tracker); +} + +#endif Modified: user/kmacy/releng_7_net_backport/sys/kern/kern_uuid.c ============================================================================== --- user/kmacy/releng_7_net_backport/sys/kern/kern_uuid.c Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/kern/kern_uuid.c Sat Mar 7 05:39:55 2009 (r189485) @@ -27,6 +27,8 @@ #include __FBSDID("$FreeBSD$"); +#include "opt_route.h" + #include #include #include @@ -37,10 +39,13 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include +#include +#include /* * See also: @@ -87,13 +92,14 @@ MTX_SYSINIT(uuid_lock, &uuid_mutex, "UUI static void uuid_node(uint16_t *node) { + INIT_VNET_NET(curvnet); struct ifnet *ifp; struct ifaddr *ifa; struct sockaddr_dl *sdl; int i; IFNET_RLOCK(); - TAILQ_FOREACH(ifp, &ifnet, if_link) { + TAILQ_FOREACH(ifp, &V_ifnet, if_link) { /* Walk the address list */ TAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) { sdl = (struct sockaddr_dl*)ifa->ifa_addr; Added: user/kmacy/releng_7_net_backport/sys/kern/subr_bufring.c ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ user/kmacy/releng_7_net_backport/sys/kern/subr_bufring.c Sat Mar 7 05:39:55 2009 (r189485) @@ -0,0 +1,68 @@ +/************************************************************************** + * + * Copyright (c) 2007,2008 Kip Macy kmacy@freebsd.org + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. The name of Kip Macy nor the names of other + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * + ***************************************************************************/ +#include +__FBSDID("$FreeBSD: head/sys/kern/subr_bufring.c 185162 2008-11-22 05:55:56Z kmacy $"); + + +#include +#include +#include +#include +#include +#include + + +struct buf_ring * +buf_ring_alloc(int count, struct malloc_type *type, int flags, struct mtx *lock) +{ + struct buf_ring *br; + + KASSERT(powerof2(count), ("buf ring must be size power of 2")); + + br = malloc(sizeof(struct buf_ring) + count*sizeof(caddr_t), + type, flags|M_ZERO); + if (br == NULL) + return (NULL); +#ifdef DEBUG_BUFRING + br->br_lock = lock; +#endif + br->br_prod_size = br->br_cons_size = count; + br->br_prod_mask = br->br_cons_mask = count-1; + br->br_prod_head = br->br_cons_head = 0; + br->br_prod_tail = br->br_cons_tail = 0; + + return (br); +} + +void +buf_ring_free(struct buf_ring *br, struct malloc_type *type) +{ + free(br, type); +} Modified: user/kmacy/releng_7_net_backport/sys/kern/subr_pcpu.c ============================================================================== --- user/kmacy/releng_7_net_backport/sys/kern/subr_pcpu.c Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/kern/subr_pcpu.c Sat Mar 7 05:39:55 2009 (r189485) @@ -56,7 +56,7 @@ __FBSDID("$FreeBSD$"); #include #include -static struct pcpu *cpuid_to_pcpu[MAXCPU]; +struct pcpu *cpuid_to_pcpu[MAXCPU]; struct cpuhead cpuhead = SLIST_HEAD_INITIALIZER(cpuhead); /* Modified: user/kmacy/releng_7_net_backport/sys/kern/vfs_export.c ============================================================================== --- user/kmacy/releng_7_net_backport/sys/kern/vfs_export.c Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/kern/vfs_export.c Sat Mar 7 05:39:55 2009 (r189485) @@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include #include @@ -235,6 +236,7 @@ vfs_free_addrlist(struct netexport *nep) if ((rnh = nep->ne_rtable[i])) { RADIX_NODE_HEAD_LOCK(rnh); (*rnh->rnh_walktree) (rnh, vfs_free_netcred, rnh); + RADIX_NODE_HEAD_UNLOCK(rnh); RADIX_NODE_HEAD_DESTROY(rnh); free(rnh, M_RTABLE); nep->ne_rtable[i] = NULL; /* not SMP safe XXX */ Modified: user/kmacy/releng_7_net_backport/sys/net/bpf.c ============================================================================== --- user/kmacy/releng_7_net_backport/sys/net/bpf.c Sat Mar 7 03:41:29 2009 (r189484) +++ user/kmacy/releng_7_net_backport/sys/net/bpf.c Sat Mar 7 05:39:55 2009 (r189485) @@ -32,10 +32,11 @@ * SUCH DAMAGE. * * @(#)bpf.c 8.4 (Berkeley) 1/9/95 - * - * $FreeBSD$ */ +#include +__FBSDID("$FreeBSD$"); + #include "opt_bpf.h" #include "opt_mac.h" #include "opt_netgraph.h" @@ -55,6 +56,7 @@ #include #include #include +#include #include #include @@ -65,9 +67,11 @@ #include #include +#include #ifdef BPF_JITTER #include #endif +#include #include #include @@ -79,7 +83,7 @@ #include -static MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); +MALLOC_DEFINE(M_BPF, "BPF", "BPF data"); #if defined(DEV_BPF) || defined(NETGRAPH_BPF) @@ -95,19 +99,17 @@ static LIST_HEAD(, bpf_if) bpf_iflist; static struct mtx bpf_mtx; /* bpf global lock */ static int bpf_bpfd_cnt; -static void bpf_allocbufs(struct bpf_d *); static void bpf_attachd(struct bpf_d *, struct bpf_if *); static void bpf_detachd(struct bpf_d *); static void bpf_freed(struct bpf_d *); -static void bpf_mcopy(const void *, void *, size_t); static int bpf_movein(struct uio *, int, struct ifnet *, struct mbuf **, struct sockaddr *, int *, struct bpf_insn *); static int bpf_setif(struct bpf_d *, struct ifreq *); static void bpf_timed_out(void *); static __inline void bpf_wakeup(struct bpf_d *); -static void catchpacket(struct bpf_d *, u_char *, u_int, - u_int, void (*)(const void *, void *, size_t), +static void catchpacket(struct bpf_d *, u_char *, u_int, u_int, + void (*)(struct bpf_d *, caddr_t, u_int, void *, u_int), struct timeval *); static void reset_d(struct bpf_d *); static int bpf_setf(struct bpf_d *, struct bpf_program *, u_long cmd); @@ -116,24 +118,19 @@ static int bpf_setdlt(struct bpf_d *, u_ static void filt_bpfdetach(struct knote *); static int filt_bpfread(struct knote *, long); static void bpf_drvinit(void *); -static void bpf_clone(void *, struct ucred *, char *, int, struct cdev **); static int bpf_stats_sysctl(SYSCTL_HANDLER_ARGS); SYSCTL_NODE(_net, OID_AUTO, bpf, CTLFLAG_RW, 0, "bpf sysctl"); -static int bpf_bufsize = 4096; -SYSCTL_INT(_net_bpf, OID_AUTO, bufsize, CTLFLAG_RW, - &bpf_bufsize, 0, "Default bpf buffer size"); -static int bpf_maxbufsize = BPF_MAXBUFSIZE; -SYSCTL_INT(_net_bpf, OID_AUTO, maxbufsize, CTLFLAG_RW, - &bpf_maxbufsize, 0, "Maximum bpf buffer size"); -static int bpf_maxinsns = BPF_MAXINSNS; +int bpf_maxinsns = BPF_MAXINSNS; SYSCTL_INT(_net_bpf, OID_AUTO, maxinsns, CTLFLAG_RW, &bpf_maxinsns, 0, "Maximum bpf program instructions"); +static int bpf_zerocopy_enable = 1; +SYSCTL_INT(_net_bpf, OID_AUTO, zerocopy_enable, CTLFLAG_RW, + &bpf_zerocopy_enable, 0, "Enable new zero-copy BPF buffer sessions"); SYSCTL_NODE(_net_bpf, OID_AUTO, stats, CTLFLAG_RW, bpf_stats_sysctl, "bpf statistics portal"); static d_open_t bpfopen; -static d_close_t bpfclose; static d_read_t bpfread; static d_write_t bpfwrite; static d_ioctl_t bpfioctl; @@ -142,9 +139,7 @@ static d_kqfilter_t bpfkqfilter; static struct cdevsw bpf_cdevsw = { .d_version = D_VERSION, - .d_flags = D_TRACKCLOSE, .d_open = bpfopen, - .d_close = bpfclose, .d_read = bpfread, .d_write = bpfwrite, .d_ioctl = bpfioctl, @@ -156,6 +151,207 @@ static struct cdevsw bpf_cdevsw = { static struct filterops bpfread_filtops = { 1, NULL, filt_bpfdetach, filt_bpfread }; +/* + * Wrapper functions for various buffering methods. If the set of buffer + * modes expands, we will probably want to introduce a switch data structure + * similar to protosw, et. + */ +static void +bpf_append_bytes(struct bpf_d *d, caddr_t buf, u_int offset, void *src, + u_int len) +{ + + BPFD_LOCK_ASSERT(d); + + switch (d->bd_bufmode) { + case BPF_BUFMODE_BUFFER: + return (bpf_buffer_append_bytes(d, buf, offset, src, len)); + + case BPF_BUFMODE_ZBUF: + d->bd_zcopy++; + return (bpf_zerocopy_append_bytes(d, buf, offset, src, len)); + + default: + panic("bpf_buf_append_bytes"); + } +} + +static void +bpf_append_mbuf(struct bpf_d *d, caddr_t buf, u_int offset, void *src, + u_int len) +{ + + BPFD_LOCK_ASSERT(d); + + switch (d->bd_bufmode) { + case BPF_BUFMODE_BUFFER: + return (bpf_buffer_append_mbuf(d, buf, offset, src, len)); + + case BPF_BUFMODE_ZBUF: + d->bd_zcopy++; + return (bpf_zerocopy_append_mbuf(d, buf, offset, src, len)); + + default: + panic("bpf_buf_append_mbuf"); + } +} + +/* + * This function gets called when the free buffer is re-assigned. + */ +static void +bpf_buf_reclaimed(struct bpf_d *d) +{ + + BPFD_LOCK_ASSERT(d); + + switch (d->bd_bufmode) { + case BPF_BUFMODE_BUFFER: + return; + + case BPF_BUFMODE_ZBUF: + bpf_zerocopy_buf_reclaimed(d); + return; + + default: + panic("bpf_buf_reclaimed"); + } +} + +/* + * If the buffer mechanism has a way to decide that a held buffer can be made + * free, then it is exposed via the bpf_canfreebuf() interface. (1) is + * returned if the buffer can be discarded, (0) is returned if it cannot. + */ +static int +bpf_canfreebuf(struct bpf_d *d) +{ + + BPFD_LOCK_ASSERT(d); + + switch (d->bd_bufmode) { + case BPF_BUFMODE_ZBUF: + return (bpf_zerocopy_canfreebuf(d)); + } + return (0); +} + +/* + * Allow the buffer model to indicate that the current store buffer is + * immutable, regardless of the appearance of space. Return (1) if the + * buffer is writable, and (0) if not. + */ +static int +bpf_canwritebuf(struct bpf_d *d) +{ + + BPFD_LOCK_ASSERT(d); + + switch (d->bd_bufmode) { + case BPF_BUFMODE_ZBUF: + return (bpf_zerocopy_canwritebuf(d)); + } + return (1); *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** From owner-svn-src-user@FreeBSD.ORG Sat Mar 7 11:06:19 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id A69F1106566B for ; Sat, 7 Mar 2009 11:06:19 +0000 (UTC) (envelope-from ivoras@gmail.com) Received: from mail-fx0-f158.google.com (mail-fx0-f158.google.com [209.85.220.158]) by mx1.freebsd.org (Postfix) with ESMTP id D26538FC2C for ; Sat, 7 Mar 2009 11:06:18 +0000 (UTC) (envelope-from ivoras@gmail.com) Received: by fxm2 with SMTP id 2so625247fxm.43 for ; Sat, 07 Mar 2009 03:06:17 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; h=domainkey-signature:mime-version:sender:received:in-reply-to :references:date:x-google-sender-auth:message-id:subject:from:to:cc :content-type:content-transfer-encoding; bh=KxK0o5mzlfB6LON1neKDgUZNJSMPpyiI2EfgMBfoHWI=; b=wTFu5pAozudslf+HOd/JomNFpJqAvy4K8AUPuk47ONVDbLw2hzxJsg/hSxSm+SoMZI /tF4DMTVUBDyzAUc23ZdzVxsfZkPpXtitvBKow9huojkfBm5iTSUj+NEWY2SM97rOHlB rsjKu3zuR3HSwoVFsTEdtvMq5gqxJfUIaJ9/0= DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; h=mime-version:sender:in-reply-to:references:date :x-google-sender-auth:message-id:subject:from:to:cc:content-type :content-transfer-encoding; b=TtJpCqjC2dNQTQI7IsdDLyzWDomrn6zKyrtMltVX8BciJn7m4mG0gPpuW5zvY5Uc/U aDhB1vIfbtBCuIJNvDs1NHU0gPI9M1vGW5mHIsFVFukaG7wRJg3u+mT0GpZP59h5vg+O tTvFP5nm3VJKfPxrI68TONIAIfvyOxKvYbG+4= MIME-Version: 1.0 Sender: ivoras@gmail.com Received: by 10.180.203.3 with SMTP id a3mr1151950bkg.146.1236422403329; Sat, 07 Mar 2009 02:40:03 -0800 (PST) In-Reply-To: <200903070118.n271IL5p034118@svn.freebsd.org> References: <200903070118.n271IL5p034118@svn.freebsd.org> Date: Sat, 7 Mar 2009 11:40:03 +0100 X-Google-Sender-Auth: 575c364f12517e30 Message-ID: <9bbcef730903070240t2068f43em9b0017b10ba992e4@mail.gmail.com> From: Ivan Voras To: Kip Macy Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: base64 Cc: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: Re: svn commit: r189470 - in user/kmacy/releng_7_net_backport/sys: amd64/include arm/include dev/bce dev/mxge i386/include ia64/include powerpc/include sparc64/include sun4v/include X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 07 Mar 2009 11:06:19 -0000 SGksCgpBcmUgdGhlc2UgbmFtZXMgdGFrZW4gZnJvbSBzb21lIHN0YW5kYXJkPyBJJ20gbm90IHN0 cm9uZ2x5IGFyZ3VpbmcKYWdhaW5zdCB0aGVtIGJ1dCBuYW1pbmcgc29tZXRoaW5nICJtYiIgYW5k IHN1Y2ggaW50cm9kdWNlcyBncmVhdApwb3RlbnRpYWwgZm9yIGNvbmZsaWN0cyB3aXRoIGxvY2Fs IGNvZGUuIChOb3QgdGhlIGFjdHVhbCBjaG9pY2Ugb2YKbGV0dGVycyBidXQgdGhhdCB0aGV5IGFy ZSB0d28gYW5kIHRocmVlLWNoYXJhY3RlcnMgbG9uZyBpZGVudGlmaWVycykuCk1heWJlIHNvbWV0 aGluZyBsaWtlIG1lbWJhciwgbWVtYmFyX3JlYWQsIG1lbWJhcl93cml0ZSB3b3VsZCBiZQpiZXR0 ZXI/CgpCdXQgSSBzZWUgdGhpcyBpcyBtZXJnZWQgZnJvbSBleGlzdGluZyBjb2RlIHNvIGl0J3Mg cHJvYmFibHkgdG9vIGxhdGUuLi4KCgoyMDA5LzMvNyBLaXAgTWFjeSA8a21hY3lAZnJlZWJzZC5v cmc+Ogo+IEF1dGhvcjoga21hY3kKPiBEYXRlOiBTYXQgTWFyIMKgNyAwMToxODoyMSAyMDA5Cj4g TmV3IFJldmlzaW9uOiAxODk0NzAKPiBVUkw6IGh0dHA6Ly9zdm4uZnJlZWJzZC5vcmcvY2hhbmdl c2V0L2Jhc2UvMTg5NDcwCj4KPiBMb2c6Cj4gwqAgbXVsdGlxIHN0ZXAgMToKPiDCoCAtIGF0b21p Y19iYXJyaWVyczogYWRkIGF0b21pYyBiYXJyaWVycyB0byBhdG9taWMuaCBvbiBhbGwgYXJjaGl0 ZWN0dXJlcwo+Cj4gTW9kaWZpZWQ6Cj4gwqB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9y dC9zeXMvYW1kNjQvaW5jbHVkZS9hdG9taWMuaAo+IMKgdXNlci9rbWFjeS9yZWxlbmdfN19uZXRf YmFja3BvcnQvc3lzL2FybS9pbmNsdWRlL2F0b21pYy5oCj4gwqB1c2VyL2ttYWN5L3JlbGVuZ183 X25ldF9iYWNrcG9ydC9zeXMvZGV2L2JjZS9pZl9iY2VyZWcuaAo+IMKgdXNlci9rbWFjeS9yZWxl bmdfN19uZXRfYmFja3BvcnQvc3lzL2Rldi9teGdlL2lmX214Z2UuYwo+IMKgdXNlci9rbWFjeS9y ZWxlbmdfN19uZXRfYmFja3BvcnQvc3lzL2Rldi9teGdlL2lmX214Z2VfdmFyLmgKPiDCoHVzZXIv a21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9pMzg2L2luY2x1ZGUvYXRvbWljLmgKPiDC oHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9pYTY0L2luY2x1ZGUvYXRvbWlj LmgKPiDCoHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9wb3dlcnBjL2luY2x1 ZGUvYXRvbWljLmgKPiDCoHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9zcGFy YzY0L2luY2x1ZGUvYXRvbWljLmgKPiDCoHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0 L3N5cy9zdW40di9pbmNsdWRlL2F0b21pYy5oCj4KPiBNb2RpZmllZDogdXNlci9rbWFjeS9yZWxl bmdfN19uZXRfYmFja3BvcnQvc3lzL2FtZDY0L2luY2x1ZGUvYXRvbWljLmgKPiA9PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT0KPiAtLS0gdXNlci9rbWFjeS9yZWxlbmdfN19uZXRfYmFja3BvcnQvc3lzL2Ft ZDY0L2luY2x1ZGUvYXRvbWljLmggU2F0IE1hciDCoDcgMDE6MTc6MTMgMjAwOSDCoCDCoCDCoCDC oChyMTg5NDY5KQo+ICsrKyB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMvYW1k NjQvaW5jbHVkZS9hdG9taWMuaCBTYXQgTWFyIMKgNyAwMToxODoyMSAyMDA5IMKgIMKgIMKgIMKg KHIxODk0NzApCj4gQEAgLTMyLDYgKzMyLDEwIEBACj4gwqAjZXJyb3IgdGhpcyBmaWxlIG5lZWRz IHN5cy9jZGVmcy5oIGFzIGEgcHJlcmVxdWlzaXRlCj4gwqAjZW5kaWYKPgo+ICsjZGVmaW5lIG1i KCkgwqAgX19hc21fXyBfX3ZvbGF0aWxlX18gKCJtZmVuY2U7IjogOiA6Im1lbW9yeSIpCj4gKyNk ZWZpbmUgd21iKCkgwqBfX2FzbV9fIF9fdm9sYXRpbGVfXyAoInNmZW5jZTsiOiA6IDoibWVtb3J5 IikKPiArI2RlZmluZSBybWIoKSDCoF9fYXNtX18gX192b2xhdGlsZV9fICgibGZlbmNlOyI6IDog OiJtZW1vcnkiKQo+ICsKPiDCoC8qCj4gwqAqIFZhcmlvdXMgc2ltcGxlIG9wZXJhdGlvbnMgb24g bWVtb3J5LCBlYWNoIG9mIHdoaWNoIGlzIGF0b21pYyBpbiB0aGUKPiDCoCogcHJlc2VuY2Ugb2Yg aW50ZXJydXB0cyBhbmQgbXVsdGlwbGUgcHJvY2Vzc29ycy4KPgo+IE1vZGlmaWVkOiB1c2VyL2tt YWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMvYXJtL2luY2x1ZGUvYXRvbWljLmgKPiA9PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT0KPiAtLS0gdXNlci9rbWFjeS9yZWxlbmdfN19uZXRfYmFja3BvcnQv c3lzL2FybS9pbmNsdWRlL2F0b21pYy5oIMKgIFNhdCBNYXIgwqA3IDAxOjE3OjEzIDIwMDkgwqAg wqAgwqAgwqAocjE4OTQ2OSkKPiArKysgdXNlci9rbWFjeS9yZWxlbmdfN19uZXRfYmFja3BvcnQv c3lzL2FybS9pbmNsdWRlL2F0b21pYy5oIMKgIFNhdCBNYXIgwqA3IDAxOjE4OjIxIDIwMDkgwqAg wqAgwqAgwqAocjE4OTQ3MCkKPiBAQCAtNDUsNiArNDUsMTAgQEAKPgo+IMKgI2luY2x1ZGUgPHN5 cy90eXBlcy5oPgo+Cj4gKyNkZWZpbmUgwqAgwqAgwqAgwqBtYigpCj4gKyNkZWZpbmUgwqAgwqAg wqAgwqB3bWIoKQo+ICsjZGVmaW5lIMKgIMKgIMKgIMKgcm1iKCkKPiArCj4gwqAjaWZuZGVmIEkz Ml9iaXQKPiDCoCNkZWZpbmUgSTMyX2JpdCAoMSA8PCA3KSDCoCDCoCDCoCDCoC8qIElSUSBkaXNh YmxlICovCj4gwqAjZW5kaWYKPgo+IE1vZGlmaWVkOiB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9i YWNrcG9ydC9zeXMvZGV2L2JjZS9pZl9iY2VyZWcuaAo+ID09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQo+ IC0tLSB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMvZGV2L2JjZS9pZl9iY2Vy ZWcuaCDCoCDCoFNhdCBNYXIgwqA3IDAxOjE3OjEzIDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ2OSkK PiArKysgdXNlci9rbWFjeS9yZWxlbmdfN19uZXRfYmFja3BvcnQvc3lzL2Rldi9iY2UvaWZfYmNl cmVnLmggwqAgwqBTYXQgTWFyIMKgNyAwMToxODoyMSAyMDA5IMKgIMKgIMKgIMKgKHIxODk0NzAp Cj4gQEAgLTU1Nyw2ICs1NTcsOCBAQAo+Cj4gwqAjZW5kaWYgLyogQkNFX0RFQlVHICovCj4KPiAr Cj4gKyNpZiBfX0ZyZWVCU0RfdmVyc2lvbiA8IDcwMTEwMQo+IMKgI2lmIGRlZmluZWQoX19pMzg2 X18pIHx8IGRlZmluZWQoX19hbWQ2NF9fKQo+IMKgI2RlZmluZSBtYigpIMKgIMKgX19hc20gdm9s YXRpbGUoIm1mZW5jZSIgOjo6ICJtZW1vcnkiKQo+IMKgI2RlZmluZSB3bWIoKSDCoCBfX2FzbSB2 b2xhdGlsZSgic2ZlbmNlIiA6OjogIm1lbW9yeSIpCj4gQEAgLTU2Niw2ICs1NjgsNyBAQAo+IMKg I2RlZmluZSBybWIoKQo+IMKgI2RlZmluZSB3bWIoKQo+IMKgI2VuZGlmCj4gKyNlbmRpZgo+Cj4g wqAvKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKioq KioqKioqKioqKioqKioqKioqKioqKi8KPiDCoC8qIERldmljZSBpZGVudGlmaWNhdGlvbiBkZWZp bml0aW9ucy4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgKi8KPgo+IE1vZGlmaWVkOiB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9y dC9zeXMvZGV2L214Z2UvaWZfbXhnZS5jCj4gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09Cj4gLS0tIHVz ZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9kZXYvbXhnZS9pZl9teGdlLmMgwqAg wqAgU2F0IE1hciDCoDcgMDE6MTc6MTMgMjAwOSDCoCDCoCDCoCDCoChyMTg5NDY5KQo+ICsrKyB1 c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMvZGV2L214Z2UvaWZfbXhnZS5jIMKg IMKgIFNhdCBNYXIgwqA3IDAxOjE4OjIxIDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ3MCkKPiBAQCAt NzQ3LDkgKzc0Nyw5IEBAIG14Z2VfbG9hZF9maXJtd2FyZV9oZWxwZXIobXhnZV9zb2Z0Y190ICoK PiDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoG14Z2VfcGlvX2NvcHkoc2MtPnNyYW0gKyBNWEdFX0ZX X09GRlNFVCArIGksCj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqBpbmZsYXRlX2J1ZmZlciArIGksCj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgwqBtaW4oMjU2VSwgKHVuc2lnbmVkKShmd19sZW4gLSBpKSkpOwo+IC0gwqAgwqAg wqAgwqAgwqAgwqAgwqAgbWIoKTsKPiArIMKgIMKgIMKgIMKgIMKgIMKgIMKgIHdtYigpOwo+IMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgZHVtbXkgPSAqc2MtPnNyYW07Cj4gLSDCoCDCoCDCoCDCoCDC oCDCoCDCoCBtYigpOwo+ICsgwqAgwqAgwqAgwqAgwqAgwqAgwqAgd21iKCk7Cj4gwqAgwqAgwqAg wqB9Cj4KPiDCoCDCoCDCoCDCoCpsaW1pdCA9IGZ3X2xlbjsKPiBAQCAtNzgyLDcgKzc4Miw3IEBA IG14Z2VfZHVtbXlfcmRtYShteGdlX3NvZnRjX3QgKnNjLCBpbnQgZW4KPiDCoCDCoCDCoCDCoC8q IGNsZWFyIGNvbmZpcm1hdGlvbiBhZGRyICovCj4gwqAgwqAgwqAgwqBjb25maXJtID0gKHZvbGF0 aWxlIHVpbnQzMl90ICopc2MtPmNtZDsKPiDCoCDCoCDCoCDCoCpjb25maXJtID0gMDsKPiAtIMKg IMKgIMKgIG1iKCk7Cj4gKyDCoCDCoCDCoCB3bWIoKTsKPgo+IMKgIMKgIMKgIMKgLyogc2VuZCBh biByZG1hIGNvbW1hbmQgdG8gdGhlIFBDSWUgZW5naW5lLCBhbmQgd2FpdCBmb3IgdGhlCj4gwqAg wqAgwqAgwqAgwqAgcmVzcG9uc2UgaW4gdGhlIGNvbmZpcm1hdGlvbiBhZGRyZXNzLiDCoFRoZSBm aXJtd2FyZSBzaG91bGQKPiBAQCAtODA0LDkgKzgwNCw5IEBAIG14Z2VfZHVtbXlfcmRtYShteGdl X3NvZnRjX3QgKnNjLCBpbnQgZW4KPiDCoCDCoCDCoCDCoHN1Ym1pdCA9ICh2b2xhdGlsZSBjaGFy ICopKHNjLT5zcmFtICsgTVhHRUZXX0JPT1RfRFVNTVlfUkRNQSk7Cj4KPiDCoCDCoCDCoCDCoG14 Z2VfcGlvX2NvcHkoc3VibWl0LCBidWYsIDY0KTsKPiAtIMKgIMKgIMKgIG1iKCk7Cj4gKyDCoCDC oCDCoCB3bWIoKTsKPiDCoCDCoCDCoCDCoERFTEFZKDEwMDApOwo+IC0gwqAgwqAgwqAgbWIoKTsK PiArIMKgIMKgIMKgIHdtYigpOwo+IMKgIMKgIMKgIMKgaSA9IDA7Cj4gwqAgwqAgwqAgwqB3aGls ZSAoKmNvbmZpcm0gIT0gMHhmZmZmZmZmZiAmJiBpIDwgMjApIHsKPiDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoERFTEFZKDEwMDApOwo+IEBAIC04NDQsNyArODQ0LDcgQEAgbXhnZV9zZW5kX2NtZCht eGdlX3NvZnRjX3QgKnNjLCB1aW50MzJfdAo+IMKgIMKgIMKgIMKgYnVmLT5yZXNwb25zZV9hZGRy LmhpZ2ggPSBodG9iZTMyKGRtYV9oaWdoKTsKPiDCoCDCoCDCoCDCoG10eF9sb2NrKCZzYy0+Y21k X210eCk7Cj4gwqAgwqAgwqAgwqByZXNwb25zZS0+cmVzdWx0ID0gMHhmZmZmZmZmZjsKPiAtIMKg IMKgIMKgIG1iKCk7Cj4gKyDCoCDCoCDCoCB3bWIoKTsKPiDCoCDCoCDCoCDCoG14Z2VfcGlvX2Nv cHkoKHZvbGF0aWxlIHZvaWQgKiljbWRfYWRkciwgYnVmLCBzaXplb2YgKCpidWYpKTsKPgo+IMKg IMKgIMKgIMKgLyogd2FpdCB1cCB0byAyMG1zICovCj4gQEAgLTg1Miw3ICs4NTIsNyBAQCBteGdl X3NlbmRfY21kKG14Z2Vfc29mdGNfdCAqc2MsIHVpbnQzMl90Cj4gwqAgwqAgwqAgwqBmb3IgKHNs ZWVwX3RvdGFsID0gMDsgc2xlZXBfdG90YWwgPCDCoDIwOyBzbGVlcF90b3RhbCsrKSB7Cj4gwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqBidXNfZG1hbWFwX3N5bmMoc2MtPmNtZF9kbWEuZG1hdCwKPiDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoHNjLT5jbWRfZG1h Lm1hcCwgQlVTX0RNQVNZTkNfUE9TVFJFQUQpOwo+IC0gwqAgwqAgwqAgwqAgwqAgwqAgwqAgbWIo KTsKPiArIMKgIMKgIMKgIMKgIMKgIMKgIMKgIHdtYigpOwo+IMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgc3dpdGNoIChiZTMydG9oKHJlc3BvbnNlLT5yZXN1bHQpKSB7Cj4gwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqBjYXNlIDA6Cj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBkYXRh LT5kYXRhMCA9IGJlMzJ0b2gocmVzcG9uc2UtPmRhdGEpOwo+IEBAIC05ODAsNyArOTgwLDcgQEAg bXhnZV9sb2FkX2Zpcm13YXJlKG14Z2Vfc29mdGNfdCAqc2MsIGludAo+IMKgIMKgIMKgIMKgLyog Y2xlYXIgY29uZmlybWF0aW9uIGFkZHIgKi8KPiDCoCDCoCDCoCDCoGNvbmZpcm0gPSAodm9sYXRp bGUgdWludDMyX3QgKilzYy0+Y21kOwo+IMKgIMKgIMKgIMKgKmNvbmZpcm0gPSAwOwo+IC0gwqAg wqAgwqAgbWIoKTsKPiArIMKgIMKgIMKgIHdtYigpOwo+IMKgIMKgIMKgIMKgLyogc2VuZCBhIHJl bG9hZCBjb21tYW5kIHRvIHRoZSBib290c3RyYXAgTUNQLCBhbmQgd2FpdCBmb3IgdGhlCj4gwqAg wqAgwqAgwqAgwqAgcmVzcG9uc2UgaW4gdGhlIGNvbmZpcm1hdGlvbiBhZGRyZXNzLiDCoFRoZSBm aXJtd2FyZSBzaG91bGQKPiDCoCDCoCDCoCDCoCDCoCB3cml0ZSBhIC0xIHRoZXJlIHRvIGluZGlj YXRlIGl0IGlzIGFsaXZlIGFuZCB3ZWxsCj4gQEAgLTEwMDUsOSArMTAwNSw5IEBAIG14Z2VfbG9h ZF9maXJtd2FyZShteGdlX3NvZnRjX3QgKnNjLCBpbnQKPgo+IMKgIMKgIMKgIMKgc3VibWl0ID0g KHZvbGF0aWxlIGNoYXIgKikoc2MtPnNyYW0gKyBNWEdFRldfQk9PVF9IQU5ET0ZGKTsKPiDCoCDC oCDCoCDCoG14Z2VfcGlvX2NvcHkoc3VibWl0LCBidWYsIDY0KTsKPiAtIMKgIMKgIMKgIG1iKCk7 Cj4gKyDCoCDCoCDCoCB3bWIoKTsKPiDCoCDCoCDCoCDCoERFTEFZKDEwMDApOwo+IC0gwqAgwqAg wqAgbWIoKTsKPiArIMKgIMKgIMKgIHdtYigpOwo+IMKgIMKgIMKgIMKgaSA9IDA7Cj4gwqAgwqAg wqAgwqB3aGlsZSAoKmNvbmZpcm0gIT0gMHhmZmZmZmZmZiAmJiBpIDwgMjApIHsKPiDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoERFTEFZKDEwMDAqMTApOwo+IEBAIC0xNjY4LDcgKzE2NjgsNyBAQCBt eGdlX3N1Ym1pdF9yZXFfYmFja3dhcmRzKG14Z2VfdHhfcmluZ190Cj4gwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgaWR4ID0gKHN0YXJ0aW5nX3Nsb3QgKyBjbnQpICYgdHgtPm1hc2s7Cj4gwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgbXhnZV9waW9fY29weSgmdHgtPmxhbmFpW2lkeF0sCj4gwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAmc3JjW2NudF0sIHNpemVvZigq c3JjKSk7Cj4gLSDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoG1iKCk7Cj4gKyDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoHdtYigpOwo+IMKgIMKgIMKgIMKgIH0KPiDCoH0KPgo+IEBAIC0xNjk0LDE0ICsx Njk0LDE0IEBAIG14Z2Vfc3VibWl0X3JlcShteGdlX3R4X3JpbmdfdCAqdHgsIG1jcF8KPgo+IMKg IMKgIMKgIMKgbGFzdF9mbGFncyA9IHNyYy0+ZmxhZ3M7Cj4gwqAgwqAgwqAgwqBzcmMtPmZsYWdz ID0gMDsKPiAtIMKgIMKgIMKgIMKgbWIoKTsKPiArIMKgIMKgIMKgIMKgd21iKCk7Cj4gwqAgwqAg wqAgwqAgZHN0ID0gZHN0cCA9ICZ0eC0+bGFuYWlbaWR4XTsKPiDCoCDCoCDCoCDCoCBzcmNwID0g c3JjOwo+Cj4gwqAgwqAgwqAgwqAgaWYgKChpZHggKyBjbnQpIDwgdHgtPm1hc2spIHsKPiDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCBmb3IgKGkgPSAwOyBpIDwgKGNudCAtIDEpOyBpICs9IDIpIHsK PiDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCBteGdlX3Bpb19jb3B5KGRzdHAs IHNyY3AsIDIgKiBzaXplb2YoKnNyYykpOwo+IC0gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqBtYigpOyAvKiBmb3JjZSB3cml0ZSBldmVyeSAzMiBieXRlcyAqLwo+ICsgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqB3bWIoKTsgLyogZm9yY2Ugd3JpdGUgZXZlcnkg MzIgYnl0ZXMgKi8KPiDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCBzcmNwICs9 IDI7Cj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgZHN0cCArPSAyOwo+IMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIH0KPiBAQCAtMTcxNCw3ICsxNzE0LDcgQEAgbXhnZV9zdWJt aXRfcmVxKG14Z2VfdHhfcmluZ190ICp0eCwgbWNwXwo+IMKgIMKgIMKgIMKgIGlmIChpIDwgY250 KSB7Cj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgLyogc3VibWl0IHRoZSBmaXJzdCByZXF1ZXN0 ICovCj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgbXhnZV9waW9fY29weShkc3RwLCBzcmNwLCBz aXplb2YoKnNyYykpOwo+IC0gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBtYigpOyAvKiBiYXJyaWVy IGJlZm9yZSBzZXR0aW5nIHZhbGlkIGZsYWcgKi8KPiArIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg d21iKCk7IC8qIGJhcnJpZXIgYmVmb3JlIHNldHRpbmcgdmFsaWQgZmxhZyAqLwo+IMKgIMKgIMKg IMKgIH0KPgo+IMKgIMKgIMKgIMKgIC8qIHJlLXdyaXRlIHRoZSBsYXN0IDMyLWJpdHMgd2l0aCB0 aGUgdmFsaWQgZmxhZ3MgKi8KPiBAQCAtMTcyNSw3ICsxNzI1LDcgQEAgbXhnZV9zdWJtaXRfcmVx KG14Z2VfdHhfcmluZ190ICp0eCwgbWNwXwo+IMKgIMKgIMKgIMKgIGRzdF9pbnRzKz0zOwo+IMKg IMKgIMKgIMKgICpkc3RfaW50cyA9IMKgKnNyY19pbnRzOwo+IMKgIMKgIMKgIMKgIHR4LT5yZXEg Kz0gY250Owo+IC0gwqAgwqAgwqAgwqBtYigpOwo+ICsgwqAgwqAgwqAgwqB3bWIoKTsKPiDCoH0K Pgo+IMKgI2lmIElGQ0FQX1RTTzQKPiBAQCAtMjE0NCwxMiArMjE0NCwxMiBAQCBteGdlX3N1Ym1p dF84cngodm9sYXRpbGUgbWNwX2tyZXFfZXRoZXJfCj4gwqAgwqAgwqAgwqBsb3cgPSBzcmMtPmFk ZHJfbG93Owo+IMKgIMKgIMKgIMKgc3JjLT5hZGRyX2xvdyA9IDB4ZmZmZmZmZmY7Cj4gwqAgwqAg wqAgwqBteGdlX3Bpb19jb3B5KGRzdCwgc3JjLCA0ICogc2l6ZW9mICgqc3JjKSk7Cj4gLSDCoCDC oCDCoCBtYigpOwo+ICsgwqAgwqAgwqAgd21iKCk7Cj4gwqAgwqAgwqAgwqBteGdlX3Bpb19jb3B5 KGRzdCArIDQsIHNyYyArIDQsIDQgKiBzaXplb2YgKCpzcmMpKTsKPiAtIMKgIMKgIMKgIG1iKCk7 Cj4gKyDCoCDCoCDCoCB3bWIoKTsKPiDCoCDCoCDCoCDCoHNyYy0+YWRkcl9sb3cgPSBsb3c7Cj4g wqAgwqAgwqAgwqBkc3QtPmFkZHJfbG93ID0gbG93Owo+IC0gwqAgwqAgwqAgbWIoKTsKPiArIMKg IMKgIMKgIHdtYigpOwo+IMKgfQo+Cj4gwqBzdGF0aWMgaW50Cj4gQEAgLTI3MzYsNyArMjczNiw3 IEBAIG14Z2VfaW50cih2b2lkICphcmcpCj4gwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqBzZW5kX2RvbmVfY291bnQgPSBiZTMydG9oKHN0YXRzLT5zZW5kX2RvbmVfY291bnQpOwo+ IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgfQo+IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgaWYgKHNj LT5sZWdhY3lfaXJxICYmIG14Z2VfZGVhc3NlcnRfd2FpdCkKPiAtIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIG1iKCk7Cj4gKyDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCB3bWIoKTsKPiDCoCDCoCDCoCDCoH0gd2hpbGUgKCooKHZvbGF0aWxlIHVpbnQ4X3QgKikgJnN0 YXRzLT52YWxpZCkpOwo+Cj4gwqAgwqAgwqAgwqBpZiAoX19wcmVkaWN0X2ZhbHNlKHN0YXRzLT5z dGF0c191cGRhdGVkKSkgewo+IEBAIC0zMzg1LDcgKzMzODUsNyBAQCBteGdlX2Nsb3NlKG14Z2Vf c29mdGNfdCAqc2MpCj4gwqAgwqAgwqAgwqBjYWxsb3V0X3N0b3AoJnNjLT5jb19oZGwpOwo+IMKg IMKgIMKgIMKgc2MtPmlmcC0+aWZfZHJ2X2ZsYWdzICY9IH5JRkZfRFJWX1JVTk5JTkc7Cj4gwqAg wqAgwqAgwqBvbGRfZG93bl9jbnQgPSBzYy0+ZG93bl9jbnQ7Cj4gLSDCoCDCoCDCoCBtYigpOwo+ ICsgwqAgwqAgwqAgd21iKCk7Cj4gwqAgwqAgwqAgwqBlcnIgPSBteGdlX3NlbmRfY21kKHNjLCBN WEdFRldfQ01EX0VUSEVSTkVUX0RPV04sICZjbWQpOwo+IMKgIMKgIMKgIMKgaWYgKGVycikgewo+ IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgZGV2aWNlX3ByaW50ZihzYy0+ZGV2LCAiQ291bGRuJ3Qg YnJpbmcgZG93biBsaW5rXG4iKTsKPiBAQCAtMzM5NCw3ICszMzk0LDcgQEAgbXhnZV9jbG9zZSht eGdlX3NvZnRjX3QgKnNjKQo+IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgLyogd2FpdCBmb3IgZG93 biBpcnEgKi8KPiDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoERFTEFZKDEwICogc2MtPmludHJfY29h bF9kZWxheSk7Cj4gwqAgwqAgwqAgwqB9Cj4gLSDCoCDCoCDCoCBtYigpOwo+ICsgwqAgwqAgwqAg d21iKCk7Cj4gwqAgwqAgwqAgwqBpZiAob2xkX2Rvd25fY250ID09IHNjLT5kb3duX2NudCkgewo+ IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgZGV2aWNlX3ByaW50ZihzYy0+ZGV2LCAibmV2ZXIgZ290 IGRvd24gaXJxXG4iKTsKPiDCoCDCoCDCoCDCoH0KPgo+IE1vZGlmaWVkOiB1c2VyL2ttYWN5L3Jl bGVuZ183X25ldF9iYWNrcG9ydC9zeXMvZGV2L214Z2UvaWZfbXhnZV92YXIuaAo+ID09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PQo+IC0tLSB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMv ZGV2L214Z2UvaWZfbXhnZV92YXIuaCBTYXQgTWFyIMKgNyAwMToxNzoxMyAyMDA5IMKgIMKgIMKg IMKgKHIxODk0NjkpCj4gKysrIHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9k ZXYvbXhnZS9pZl9teGdlX3Zhci5oIFNhdCBNYXIgwqA3IDAxOjE4OjIxIDIwMDkgwqAgwqAgwqAg wqAocjE4OTQ3MCkKPiBAQCAtMjgyLDYgKzI4Miw4IEBAIHN0cnVjdCBteGdlX21lZGlhX3R5cGUK PiDCoC8qIGltcGxlbWVudCBvdXIgb3duIG1lbW9yeSBiYXJyaWVycywgc2luY2UgYnVzX3NwYWNl X2JhcnJpZXIKPiDCoCDCoGNhbm5vdCBoYW5kbGUgd3JpdGUtY29tYmluaW5nIHJlZ2lvbnMgKi8K Pgo+ICsjaWYgX19GcmVlQlNEX3ZlcnNpb24gPCA3MDExMDEKPiArCj4gwqAjaWYgZGVmaW5lZCAo X19HTlVDX18pCj4gwqAgI2lmICNjcHUoaTM4NikgfHwgZGVmaW5lZCBfX2kzODYgfHwgZGVmaW5l ZCBpMzg2IHx8IGRlZmluZWQgX19pMzg2X18gfHwgI2NwdSh4ODZfNjQpIHx8IGRlZmluZWQgX194 ODZfNjRfXwo+IMKgIMKgICNkZWZpbmUgbWIoKSDCoF9fYXNtX18gX192b2xhdGlsZV9fICgic2Zl bmNlOyI6IDogOiJtZW1vcnkiKQo+IEBAIC0yOTYsNiArMjk4LDggQEAgc3RydWN0IG14Z2VfbWVk aWFfdHlwZQo+IMKgICNlcnJvciAidW5rbm93biBjb21waWxlciIKPiDCoCNlbmRpZgo+Cj4gKyNl bmRpZgo+ICsKPiDCoHN0YXRpYyBpbmxpbmUgdm9pZAo+IMKgbXhnZV9waW9fY29weSh2b2xhdGls ZSB2b2lkICp0b192LCB2b2lkICpmcm9tX3YsIHNpemVfdCBzaXplKQo+IMKgewo+Cj4gTW9kaWZp ZWQ6IHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9pMzg2L2luY2x1ZGUvYXRv bWljLmgKPiA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KPiAtLS0gdXNlci9rbWFjeS9yZWxlbmdfN19u ZXRfYmFja3BvcnQvc3lzL2kzODYvaW5jbHVkZS9hdG9taWMuaCDCoFNhdCBNYXIgwqA3IDAxOjE3 OjEzIDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ2OSkKPiArKysgdXNlci9rbWFjeS9yZWxlbmdfN19u ZXRfYmFja3BvcnQvc3lzL2kzODYvaW5jbHVkZS9hdG9taWMuaCDCoFNhdCBNYXIgwqA3IDAxOjE4 OjIxIDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ3MCkKPiBAQCAtMzIsNiArMzIsNDcgQEAKPiDCoCNl cnJvciB0aGlzIGZpbGUgbmVlZHMgc3lzL2NkZWZzLmggYXMgYSBwcmVyZXF1aXNpdGUKPiDCoCNl bmRpZgo+Cj4gKyNpZiBkZWZpbmVkKF9LRVJORUwpCj4gKyNpbmNsdWRlIDxtYWNoaW5lL2NwdWZ1 bmMuaD4KPiArI2luY2x1ZGUgPG1hY2hpbmUvc3BlY2lhbHJlZy5oPgo+ICsjZGVmaW5lIMKgIMKg IMKgIMKgbWIoKSDCoCDCoF9fYXNtIF9fdm9sYXRpbGUoIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgIMKgIFwKPiArIMKgIMKgIMKgICJ0ZXN0bCDCoCUwLGNwdV9mZWF0dXJlIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5cCj4gKyDCoCDCoCDCoCBqZSDCoCDCoCDC oDJmIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgXG5cCj4gKyDCoCDCoCDCoCBtZmVuY2UgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBcblwKPiArMTogwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgXG5cCj4gKyDCoCDCoCDCoCAuc2VjdGlvbiAudGV4dC5vZmZwYXRoIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5cCj4gKzI6IMKgIMKgIGxvY2sgwqAgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBc blwKPiArIMKgIMKgIMKgIGFkZGwgwqAgwqAkMCxjcHVfZmVhdHVyZSDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoFxuXAo+ICsgwqAgwqAgwqAgam1wIMKgIMKgIDFiIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5cCj4g KyDCoCDCoCDCoCAudGV4dCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCBcblwKPiArIiDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoFwK PiArIMKgIMKgIMKgIDogOiAiaSIoQ1BVSURfU1NFMikgOiAibWVtb3J5IikKPiArI2RlZmluZSDC oCDCoCDCoCDCoHdtYigpIMKgIF9fYXNtIF9fdm9sYXRpbGUoIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIFwKPiArIMKgIMKgIMKgICJ0ZXN0bCDCoCUwLGNwdV9mZWF0dXJlIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5cCj4gKyDCoCDCoCDCoCBqZSDCoCDC oCDCoDJmIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgXG5cCj4gKyDCoCDCoCDCoCBzZmVuY2UgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBcblwKPiArMTogwqAgwqAgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgXG5cCj4gKyDCoCDCoCDCoCAuc2VjdGlvbiAudGV4dC5vZmZwYXRoIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5cCj4gKzI6IMKgIMKgIGxvY2sgwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqBcblwKPiArIMKgIMKgIMKgIGFkZGwgwqAgwqAkMCxjcHVfZmVhdHVyZSDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoFxuXAo+ICsgwqAgwqAgwqAgam1wIMKgIMKgIDFiIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5c Cj4gKyDCoCDCoCDCoCAudGV4dCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCBcblwKPiArIiDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oFwKPiArIMKgIMKgIMKgIDogOiAiaSIoQ1BVSURfWE1NKSA6ICJtZW1vcnkiKQo+ICsjZGVmaW5l IMKgIMKgIMKgIMKgcm1iKCkgwqAgX19hc20gX192b2xhdGlsZSggwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqAgwqAgwqAgXAo+ICsgwqAgwqAgwqAgInRlc3RsIMKgJTAsY3B1X2ZlYXR1cmUgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBcblwKPiArIMKgIMKgIMKgIGplIMKg IMKgIMKgMmYgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAg wqAgwqAgwqBcblwKPiArIMKgIMKgIMKgIGxmZW5jZSDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoFxuXAo+ICsxOiDCoCDCoCDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoCDCoCDCoCBcblwKPiArIMKgIMKgIMKgIC5zZWN0aW9uIC50ZXh0Lm9mZnBhdGggwqAgwqAg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBcblwKPiArMjogwqAgwqAgbG9jayDCoCDC oCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDCoCDC oCDCoFxuXAo+ICsgwqAgwqAgwqAgYWRkbCDCoCDCoCQwLGNwdV9mZWF0dXJlIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXG5cCj4gKyDCoCDCoCDCoCBqbXAgwqAgwqAgMWIg wqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqAgwqBc blwKPiArIMKgIMKgIMKgIC50ZXh0IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIFxuXAo+ICsiIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgXAo+ICsgwqAgwqAgwqAgOiA6ICJpIihDUFVJRF9TU0UyKSA6ICJtZW1vcnkiKQo+ICsjZW5k aWYKPiArCj4gwqAvKgo+IMKgKiBWYXJpb3VzIHNpbXBsZSBvcGVyYXRpb25zIG9uIG1lbW9yeSwg ZWFjaCBvZiB3aGljaCBpcyBhdG9taWMgaW4gdGhlCj4gwqAqIHByZXNlbmNlIG9mIGludGVycnVw dHMgYW5kIG11bHRpcGxlIHByb2Nlc3NvcnMuCj4KPiBNb2RpZmllZDogdXNlci9rbWFjeS9yZWxl bmdfN19uZXRfYmFja3BvcnQvc3lzL2lhNjQvaW5jbHVkZS9hdG9taWMuaAo+ID09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PQo+IC0tLSB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMvaWE2 NC9pbmNsdWRlL2F0b21pYy5oIMKgU2F0IE1hciDCoDcgMDE6MTc6MTMgMjAwOSDCoCDCoCDCoCDC oChyMTg5NDY5KQo+ICsrKyB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9zeXMvaWE2 NC9pbmNsdWRlL2F0b21pYy5oIMKgU2F0IE1hciDCoDcgMDE6MTg6MjEgMjAwOSDCoCDCoCDCoCDC oChyMTg5NDcwKQo+IEBAIC0yOSw2ICsyOSwxMCBAQAo+IMKgI2lmbmRlZiBfTUFDSElORV9BVE9N SUNfSF8KPiDCoCNkZWZpbmUgwqAgwqAgwqAgwqBfTUFDSElORV9BVE9NSUNfSF8KPgo+ICsjZGVm aW5lIMKgIMKgIMKgIMKgbWIoKQo+ICsjZGVmaW5lIMKgIMKgIMKgIMKgd21iKCkKPiArI2RlZmlu ZSDCoCDCoCDCoCDCoHJtYigpCj4gKwo+IMKgLyoKPiDCoCogVmFyaW91cyBzaW1wbGUgYXJpdGht ZXRpYyBvbiBtZW1vcnkgd2hpY2ggaXMgYXRvbWljIGluIHRoZSBwcmVzZW5jZQo+IMKgKiBvZiBp bnRlcnJ1cHRzIGFuZCBTTVAgc2FmZS4KPgo+IE1vZGlmaWVkOiB1c2VyL2ttYWN5L3JlbGVuZ183 X25ldF9iYWNrcG9ydC9zeXMvcG93ZXJwYy9pbmNsdWRlL2F0b21pYy5oCj4gPT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09Cj4gLS0tIHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9wb3dl cnBjL2luY2x1ZGUvYXRvbWljLmggwqAgwqAgwqAgU2F0IE1hciDCoDcgMDE6MTc6MTMgMjAwOSDC oCDCoCDCoCDCoChyMTg5NDY5KQo+ICsrKyB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9y dC9zeXMvcG93ZXJwYy9pbmNsdWRlL2F0b21pYy5oIMKgIMKgIMKgIFNhdCBNYXIgwqA3IDAxOjE4 OjIxIDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ3MCkKPiBAQCAtMzksNiArMzksMTAgQEAKPiDCoCNk ZWZpbmUgwqAgwqAgwqAgwqBfX0FUT01JQ19CQVJSSUVSIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKg IMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgIMKgXAo+IMKgIMKgIF9fYXNtIF9fdm9s YXRpbGUoInN5bmMiIDogOiA6ICJtZW1vcnkiKQo+Cj4gKyNkZWZpbmUgbWIoKSDCoCBfX0FUT01J Q19CQVJSSUVSCj4gKyNkZWZpbmUgwqAgwqAgwqAgwqB3bWIoKSDCoCBtYigpCj4gKyNkZWZpbmUg wqAgwqAgwqAgwqBybWIoKSDCoCBtYigpCj4gKwo+IMKgLyoKPiDCoCogYXRvbWljX2FkZChwLCB2 KQo+IMKgKiB7ICpwICs9IHY7IH0KPgo+IE1vZGlmaWVkOiB1c2VyL2ttYWN5L3JlbGVuZ183X25l dF9iYWNrcG9ydC9zeXMvc3BhcmM2NC9pbmNsdWRlL2F0b21pYy5oCj4gPT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09Cj4gLS0tIHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9zcGFyYzY0 L2luY2x1ZGUvYXRvbWljLmggwqAgwqAgwqAgU2F0IE1hciDCoDcgMDE6MTc6MTMgMjAwOSDCoCDC oCDCoCDCoChyMTg5NDY5KQo+ICsrKyB1c2VyL2ttYWN5L3JlbGVuZ183X25ldF9iYWNrcG9ydC9z eXMvc3BhcmM2NC9pbmNsdWRlL2F0b21pYy5oIMKgIMKgIMKgIFNhdCBNYXIgwqA3IDAxOjE4OjIx IDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ3MCkKPiBAQCAtNDAsNiArNDAsMTAgQEAKPiDCoCNkZWZp bmUgwqAgwqAgwqAgwqBfX0FTSV9BVE9NSUMgwqAgwqBBU0lfUAo+IMKgI2VuZGlmCj4KPiArI2Rl ZmluZSBtYigpIMKgIF9fYXNtX18gX192b2xhdGlsZV9fICgibWVtYmFyICNNZW1Jc3N1ZSI6IDog OiJtZW1vcnkiKQo+ICsjZGVmaW5lIHdtYigpIMKgbWIoKQo+ICsjZGVmaW5lIHJtYigpIMKgbWIo KQo+ICsKPiDCoC8qCj4gwqAqIFZhcmlvdXMgc2ltcGxlIGFyaXRobWV0aWMgb24gbWVtb3J5IHdo aWNoIGlzIGF0b21pYyBpbiB0aGUgcHJlc2VuY2UKPiDCoCogb2YgaW50ZXJydXB0cyBhbmQgbXVs dGlwbGUgcHJvY2Vzc29ycy4gwqBTZWUgYXRvbWljKDkpIGZvciBkZXRhaWxzLgo+Cj4gTW9kaWZp ZWQ6IHVzZXIva21hY3kvcmVsZW5nXzdfbmV0X2JhY2twb3J0L3N5cy9zdW40di9pbmNsdWRlL2F0 b21pYy5oCj4gPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09 PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09Cj4gLS0tIHVzZXIva21hY3kvcmVsZW5nXzdf bmV0X2JhY2twb3J0L3N5cy9zdW40di9pbmNsdWRlL2F0b21pYy5oIFNhdCBNYXIgwqA3IDAxOjE3 OjEzIDIwMDkgwqAgwqAgwqAgwqAocjE4OTQ2OSkKPiArKysgdXNlci9rbWFjeS9yZWxlbmdfN19u ZXRfYmFja3BvcnQvc3lzL3N1bjR2L2luY2x1ZGUvYXRvbWljLmggU2F0IE1hciDCoDcgMDE6MTg6 MjEgMjAwOSDCoCDCoCDCoCDCoChyMTg5NDcwKQo+IEBAIC0zMyw2ICszMywxMCBAQAo+Cj4gwqAj aW5jbHVkZSA8bWFjaGluZS9jcHVmdW5jLmg+Cj4KPiArI2RlZmluZSBtYigpIMKgIF9fYXNtX18g X192b2xhdGlsZV9fICgibWVtYmFyICNNZW1Jc3N1ZSI6IDogOiJtZW1vcnkiKQo+ICsjZGVmaW5l IHdtYigpIMKgbWIoKQo+ICsjZGVmaW5lIHJtYigpIMKgbWIoKQo+ICsKPiDCoC8qIFVzZXJsYW5k IG5lZWRzIGRpZmZlcmVudCBBU0kncy4gKi8KPiDCoCNpZmRlZiBfS0VSTkVMCj4gwqAjZGVmaW5l IMKgIMKgIMKgIMKgX19BU0lfQVRPTUlDIMKgIMKgQVNJX04KPgo= From owner-svn-src-user@FreeBSD.ORG Sat Mar 7 20:11:46 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id C1956106566C; Sat, 7 Mar 2009 20:11:46 +0000 (UTC) (envelope-from mat.macy@gmail.com) Received: from rv-out-0506.google.com (rv-out-0506.google.com [209.85.198.225]) by mx1.freebsd.org (Postfix) with ESMTP id 88D668FC14; Sat, 7 Mar 2009 20:11:46 +0000 (UTC) (envelope-from mat.macy@gmail.com) Received: by rv-out-0506.google.com with SMTP id f6so1078966rvb.43 for ; Sat, 07 Mar 2009 12:11:46 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; h=domainkey-signature:mime-version:sender:received:in-reply-to :references:date:x-google-sender-auth:message-id:subject:from:to:cc :content-type:content-transfer-encoding; bh=1EklUfZbrWkq5DrMLQzCZNqJiU9GLpqiyUNd61u2wwQ=; b=HmzVOguGHQfNG35nX2N/PA8C2tkLNdCrR7hBBFLe4WRP9QDu49SbJNVjOmpKWMshzb TcHGDW7d3mvKIVVw9LbKLw54zu6F2Tb4Q8dEZ25ZGJXNkilQ6yH5uOHEYCFhes8omQdS +I/DjWl9TN/BWTg4fugMK3NkG77MdWg86bwzA= DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; h=mime-version:sender:in-reply-to:references:date :x-google-sender-auth:message-id:subject:from:to:cc:content-type :content-transfer-encoding; b=LPEcRP7ONtPbk3U65TFW4C+EPa9E9TUMdw1LuiLHh5n+MXLkHxnhaOA1DQt0rtG0TW 7/U19wqgEp6blNzuKU3NJC7IoDWDfahgGsTqo0/PzHNobagWNNDsIAsrhp/nQ93TudsF bnkqKEp98L74k4JZOsvd7qBxg8gliYXXecp88= MIME-Version: 1.0 Sender: mat.macy@gmail.com Received: by 10.140.199.15 with SMTP id w15mr2050610rvf.253.1236456706134; Sat, 07 Mar 2009 12:11:46 -0800 (PST) In-Reply-To: <9bbcef730903070240t2068f43em9b0017b10ba992e4@mail.gmail.com> References: <200903070118.n271IL5p034118@svn.freebsd.org> <9bbcef730903070240t2068f43em9b0017b10ba992e4@mail.gmail.com> Date: Sat, 7 Mar 2009 12:11:46 -0800 X-Google-Sender-Auth: efb3f3f1ccb0ef31 Message-ID: <3c1674c90903071211y29ea8ee6u412352faf9cd108f@mail.gmail.com> From: Kip Macy To: Ivan Voras Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable Cc: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: Re: svn commit: r189470 - in user/kmacy/releng_7_net_backport/sys: amd64/include arm/include dev/bce dev/mxge i386/include ia64/include powerpc/include sparc64/include sun4v/include X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 07 Mar 2009 20:11:47 -0000 The names are what are used in Linux. There are a number of drivers in the tree that share code with Linux that #define them in a platform dependent way. -Kip On Sat, Mar 7, 2009 at 2:40 AM, Ivan Voras wrote: > Hi, > > Are these names taken from some standard? I'm not strongly arguing > against them but naming something "mb" and such introduces great > potential for conflicts with local code. (Not the actual choice of > letters but that they are two and three-characters long identifiers). > Maybe something like membar, membar_read, membar_write would be > better? > > But I see this is merged from existing code so it's probably too late... > > > 2009/3/7 Kip Macy : >> Author: kmacy >> Date: Sat Mar =A07 01:18:21 2009 >> New Revision: 189470 >> URL: http://svn.freebsd.org/changeset/base/189470 >> >> Log: >> =A0 multiq step 1: >> =A0 - atomic_barriers: add atomic barriers to atomic.h on all architectu= res >> >> Modified: >> =A0user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h >> =A0user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h >> =A0user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h >> =A0user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c >> =A0user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h >> =A0user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h >> =A0user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h >> =A0user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h >> =A0user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h >> =A0user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h >> >> Modified: user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h Sat Mar = =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h Sat Mar = =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -32,6 +32,10 @@ >> =A0#error this file needs sys/cdefs.h as a prerequisite >> =A0#endif >> >> +#define mb() =A0 __asm__ __volatile__ ("mfence;": : :"memory") >> +#define wmb() =A0__asm__ __volatile__ ("sfence;": : :"memory") >> +#define rmb() =A0__asm__ __volatile__ ("lfence;": : :"memory") >> + >> =A0/* >> =A0* Various simple operations on memory, each of which is atomic in the >> =A0* presence of interrupts and multiple processors. >> >> Modified: user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h =A0 Sat Ma= r =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h =A0 Sat Ma= r =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -45,6 +45,10 @@ >> >> =A0#include >> >> +#define =A0 =A0 =A0 =A0mb() >> +#define =A0 =A0 =A0 =A0wmb() >> +#define =A0 =A0 =A0 =A0rmb() >> + >> =A0#ifndef I32_bit >> =A0#define I32_bit (1 << 7) =A0 =A0 =A0 =A0/* IRQ disable */ >> =A0#endif >> >> Modified: user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h =A0 =A0Sat = Mar =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h =A0 =A0Sat = Mar =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -557,6 +557,8 @@ >> >> =A0#endif /* BCE_DEBUG */ >> >> + >> +#if __FreeBSD_version < 701101 >> =A0#if defined(__i386__) || defined(__amd64__) >> =A0#define mb() =A0 =A0__asm volatile("mfence" ::: "memory") >> =A0#define wmb() =A0 __asm volatile("sfence" ::: "memory") >> @@ -566,6 +568,7 @@ >> =A0#define rmb() >> =A0#define wmb() >> =A0#endif >> +#endif >> >> =A0/********************************************************************= ********/ >> =A0/* Device identification definitions. =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 */ >> >> Modified: user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c =A0 =A0 Sat = Mar =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c =A0 =A0 Sat = Mar =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -747,9 +747,9 @@ mxge_load_firmware_helper(mxge_softc_t * >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0mxge_pio_copy(sc->sram + MXGE_FW_OFFSET += i, >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0inflate_buffe= r + i, >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0min(256U, (un= signed)(fw_len - i))); >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0dummy =3D *sc->sram; >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0} >> >> =A0 =A0 =A0 =A0*limit =3D fw_len; >> @@ -782,7 +782,7 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en >> =A0 =A0 =A0 =A0/* clear confirmation addr */ >> =A0 =A0 =A0 =A0confirm =3D (volatile uint32_t *)sc->cmd; >> =A0 =A0 =A0 =A0*confirm =3D 0; >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> >> =A0 =A0 =A0 =A0/* send an rdma command to the PCIe engine, and wait for = the >> =A0 =A0 =A0 =A0 =A0 response in the confirmation address. =A0The firmwar= e should >> @@ -804,9 +804,9 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en >> =A0 =A0 =A0 =A0submit =3D (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY= _RDMA); >> >> =A0 =A0 =A0 =A0mxge_pio_copy(submit, buf, 64); >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0DELAY(1000); >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0i =3D 0; >> =A0 =A0 =A0 =A0while (*confirm !=3D 0xffffffff && i < 20) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0DELAY(1000); >> @@ -844,7 +844,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t >> =A0 =A0 =A0 =A0buf->response_addr.high =3D htobe32(dma_high); >> =A0 =A0 =A0 =A0mtx_lock(&sc->cmd_mtx); >> =A0 =A0 =A0 =A0response->result =3D 0xffffffff; >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*bu= f)); >> >> =A0 =A0 =A0 =A0/* wait up to 20ms */ >> @@ -852,7 +852,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t >> =A0 =A0 =A0 =A0for (sleep_total =3D 0; sleep_total < =A020; sleep_total+= +) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0bus_dmamap_sync(sc->cmd_dma.dmat, >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0sc->cmd_d= ma.map, BUS_DMASYNC_POSTREAD); >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0switch (be32toh(response->result)) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0case 0: >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0data->data0 =3D be32toh(r= esponse->data); >> @@ -980,7 +980,7 @@ mxge_load_firmware(mxge_softc_t *sc, int >> =A0 =A0 =A0 =A0/* clear confirmation addr */ >> =A0 =A0 =A0 =A0confirm =3D (volatile uint32_t *)sc->cmd; >> =A0 =A0 =A0 =A0*confirm =3D 0; >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0/* send a reload command to the bootstrap MCP, and wait f= or the >> =A0 =A0 =A0 =A0 =A0 response in the confirmation address. =A0The firmwar= e should >> =A0 =A0 =A0 =A0 =A0 write a -1 there to indicate it is alive and well >> @@ -1005,9 +1005,9 @@ mxge_load_firmware(mxge_softc_t *sc, int >> >> =A0 =A0 =A0 =A0submit =3D (volatile char *)(sc->sram + MXGEFW_BOOT_HANDO= FF); >> =A0 =A0 =A0 =A0mxge_pio_copy(submit, buf, 64); >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0DELAY(1000); >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0i =3D 0; >> =A0 =A0 =A0 =A0while (*confirm !=3D 0xffffffff && i < 20) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0DELAY(1000*10); >> @@ -1668,7 +1668,7 @@ mxge_submit_req_backwards(mxge_tx_ring_t >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 idx =3D (starting_slot + cnt) & tx->mask= ; >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 mxge_pio_copy(&tx->lanai[idx], >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0&src[cnt], si= zeof(*src)); >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0mb(); >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0wmb(); >> =A0 =A0 =A0 =A0 } >> =A0} >> >> @@ -1694,14 +1694,14 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ >> >> =A0 =A0 =A0 =A0last_flags =3D src->flags; >> =A0 =A0 =A0 =A0src->flags =3D 0; >> - =A0 =A0 =A0 =A0mb(); >> + =A0 =A0 =A0 =A0wmb(); >> =A0 =A0 =A0 =A0 dst =3D dstp =3D &tx->lanai[idx]; >> =A0 =A0 =A0 =A0 srcp =3D src; >> >> =A0 =A0 =A0 =A0 if ((idx + cnt) < tx->mask) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 for (i =3D 0; i < (cnt - 1); i +=3D 2) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 mxge_pio_copy(dstp, srcp= , 2 * sizeof(*src)); >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0mb(); /* force write ev= ery 32 bytes */ >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0wmb(); /* force write e= very 32 bytes */ >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 srcp +=3D 2; >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 dstp +=3D 2; >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 } >> @@ -1714,7 +1714,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ >> =A0 =A0 =A0 =A0 if (i < cnt) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 /* submit the first request */ >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 mxge_pio_copy(dstp, srcp, sizeof(*src)); >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0mb(); /* barrier before setting valid f= lag */ >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0wmb(); /* barrier before setting valid = flag */ >> =A0 =A0 =A0 =A0 } >> >> =A0 =A0 =A0 =A0 /* re-write the last 32-bits with the valid flags */ >> @@ -1725,7 +1725,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ >> =A0 =A0 =A0 =A0 dst_ints+=3D3; >> =A0 =A0 =A0 =A0 *dst_ints =3D =A0*src_ints; >> =A0 =A0 =A0 =A0 tx->req +=3D cnt; >> - =A0 =A0 =A0 =A0mb(); >> + =A0 =A0 =A0 =A0wmb(); >> =A0} >> >> =A0#if IFCAP_TSO4 >> @@ -2144,12 +2144,12 @@ mxge_submit_8rx(volatile mcp_kreq_ether_ >> =A0 =A0 =A0 =A0low =3D src->addr_low; >> =A0 =A0 =A0 =A0src->addr_low =3D 0xffffffff; >> =A0 =A0 =A0 =A0mxge_pio_copy(dst, src, 4 * sizeof (*src)); >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0src->addr_low =3D low; >> =A0 =A0 =A0 =A0dst->addr_low =3D low; >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0} >> >> =A0static int >> @@ -2736,7 +2736,7 @@ mxge_intr(void *arg) >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0send_done_count =3D be32t= oh(stats->send_done_count); >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0} >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0if (sc->legacy_irq && mxge_deassert_wait) >> - =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0} while (*((volatile uint8_t *) &stats->valid)); >> >> =A0 =A0 =A0 =A0if (__predict_false(stats->stats_updated)) { >> @@ -3385,7 +3385,7 @@ mxge_close(mxge_softc_t *sc) >> =A0 =A0 =A0 =A0callout_stop(&sc->co_hdl); >> =A0 =A0 =A0 =A0sc->ifp->if_drv_flags &=3D ~IFF_DRV_RUNNING; >> =A0 =A0 =A0 =A0old_down_cnt =3D sc->down_cnt; >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0err =3D mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd)= ; >> =A0 =A0 =A0 =A0if (err) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0device_printf(sc->dev, "Couldn't bring do= wn link\n"); >> @@ -3394,7 +3394,7 @@ mxge_close(mxge_softc_t *sc) >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0/* wait for down irq */ >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0DELAY(10 * sc->intr_coal_delay); >> =A0 =A0 =A0 =A0} >> - =A0 =A0 =A0 mb(); >> + =A0 =A0 =A0 wmb(); >> =A0 =A0 =A0 =A0if (old_down_cnt =3D=3D sc->down_cnt) { >> =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0device_printf(sc->dev, "never got down ir= q\n"); >> =A0 =A0 =A0 =A0} >> >> Modified: user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h Sat Mar = =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h Sat Mar = =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -282,6 +282,8 @@ struct mxge_media_type >> =A0/* implement our own memory barriers, since bus_space_barrier >> =A0 =A0cannot handle write-combining regions */ >> >> +#if __FreeBSD_version < 701101 >> + >> =A0#if defined (__GNUC__) >> =A0 #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__= || #cpu(x86_64) || defined __x86_64__ >> =A0 =A0 #define mb() =A0__asm__ __volatile__ ("sfence;": : :"memory") >> @@ -296,6 +298,8 @@ struct mxge_media_type >> =A0 #error "unknown compiler" >> =A0#endif >> >> +#endif >> + >> =A0static inline void >> =A0mxge_pio_copy(volatile void *to_v, void *from_v, size_t size) >> =A0{ >> >> Modified: user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h =A0Sat Ma= r =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h =A0Sat Ma= r =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -32,6 +32,47 @@ >> =A0#error this file needs sys/cdefs.h as a prerequisite >> =A0#endif >> >> +#if defined(_KERNEL) >> +#include >> +#include >> +#define =A0 =A0 =A0 =A0mb() =A0 =A0__asm __volatile( =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 \ >> + =A0 =A0 =A0 "testl =A0%0,cpu_feature =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 je =A0 =A0 =A02f =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 mfence =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> +1: =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 \n\ >> + =A0 =A0 =A0 .section .text.offpath =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0\n\ >> +2: =A0 =A0 lock =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 addl =A0 =A0$0,cpu_feature =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 jmp =A0 =A0 1b =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 .text =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 \n\ >> +" =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\ >> + =A0 =A0 =A0 : : "i"(CPUID_SSE2) : "memory") >> +#define =A0 =A0 =A0 =A0wmb() =A0 __asm __volatile( =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 \ >> + =A0 =A0 =A0 "testl =A0%0,cpu_feature =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 je =A0 =A0 =A02f =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 sfence =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> +1: =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 \n\ >> + =A0 =A0 =A0 .section .text.offpath =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0\n\ >> +2: =A0 =A0 lock =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 addl =A0 =A0$0,cpu_feature =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 jmp =A0 =A0 1b =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 .text =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 \n\ >> +" =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\ >> + =A0 =A0 =A0 : : "i"(CPUID_XMM) : "memory") >> +#define =A0 =A0 =A0 =A0rmb() =A0 __asm __volatile( =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 \ >> + =A0 =A0 =A0 "testl =A0%0,cpu_feature =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 je =A0 =A0 =A02f =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 lfence =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> +1: =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 \n\ >> + =A0 =A0 =A0 .section .text.offpath =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0\n\ >> +2: =A0 =A0 lock =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 addl =A0 =A0$0,cpu_feature =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 jmp =A0 =A0 1b =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0= =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\n\ >> + =A0 =A0 =A0 .text =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 \n\ >> +" =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\ >> + =A0 =A0 =A0 : : "i"(CPUID_SSE2) : "memory") >> +#endif >> + >> =A0/* >> =A0* Various simple operations on memory, each of which is atomic in the >> =A0* presence of interrupts and multiple processors. >> >> Modified: user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h =A0Sat Ma= r =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h =A0Sat Ma= r =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -29,6 +29,10 @@ >> =A0#ifndef _MACHINE_ATOMIC_H_ >> =A0#define =A0 =A0 =A0 =A0_MACHINE_ATOMIC_H_ >> >> +#define =A0 =A0 =A0 =A0mb() >> +#define =A0 =A0 =A0 =A0wmb() >> +#define =A0 =A0 =A0 =A0rmb() >> + >> =A0/* >> =A0* Various simple arithmetic on memory which is atomic in the presence >> =A0* of interrupts and SMP safe. >> >> Modified: user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h =A0 = =A0 =A0 Sat Mar =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h =A0 = =A0 =A0 Sat Mar =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -39,6 +39,10 @@ >> =A0#define =A0 =A0 =A0 =A0__ATOMIC_BARRIER =A0 =A0 =A0 =A0 =A0 =A0 =A0 = =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0 =A0\ >> =A0 =A0 __asm __volatile("sync" : : : "memory") >> >> +#define mb() =A0 __ATOMIC_BARRIER >> +#define =A0 =A0 =A0 =A0wmb() =A0 mb() >> +#define =A0 =A0 =A0 =A0rmb() =A0 mb() >> + >> =A0/* >> =A0* atomic_add(p, v) >> =A0* { *p +=3D v; } >> >> Modified: user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h =A0 = =A0 =A0 Sat Mar =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h =A0 = =A0 =A0 Sat Mar =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -40,6 +40,10 @@ >> =A0#define =A0 =A0 =A0 =A0__ASI_ATOMIC =A0 =A0ASI_P >> =A0#endif >> >> +#define mb() =A0 __asm__ __volatile__ ("membar #MemIssue": : :"memory") >> +#define wmb() =A0mb() >> +#define rmb() =A0mb() >> + >> =A0/* >> =A0* Various simple arithmetic on memory which is atomic in the presence >> =A0* of interrupts and multiple processors. =A0See atomic(9) for details= . >> >> Modified: user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h >> =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D >> --- user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Sat Mar = =A07 01:17:13 2009 =A0 =A0 =A0 =A0(r189469) >> +++ user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Sat Mar = =A07 01:18:21 2009 =A0 =A0 =A0 =A0(r189470) >> @@ -33,6 +33,10 @@ >> >> =A0#include >> >> +#define mb() =A0 __asm__ __volatile__ ("membar #MemIssue": : :"memory") >> +#define wmb() =A0mb() >> +#define rmb() =A0mb() >> + >> =A0/* Userland needs different ASI's. */ >> =A0#ifdef _KERNEL >> =A0#define =A0 =A0 =A0 =A0__ASI_ATOMIC =A0 =A0ASI_N >> > From owner-svn-src-user@FreeBSD.ORG Sat Mar 7 22:19:11 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 806DE1065676; Sat, 7 Mar 2009 22:19:11 +0000 (UTC) (envelope-from rwatson@FreeBSD.org) Received: from cyrus.watson.org (cyrus.watson.org [65.122.17.42]) by mx1.freebsd.org (Postfix) with ESMTP id 32B2E8FC1A; Sat, 7 Mar 2009 22:19:11 +0000 (UTC) (envelope-from rwatson@FreeBSD.org) Received: from fledge.watson.org (fledge.watson.org [65.122.17.41]) by cyrus.watson.org (Postfix) with ESMTPS id B618D46B65; Sat, 7 Mar 2009 17:19:10 -0500 (EST) Date: Sat, 7 Mar 2009 22:19:10 +0000 (GMT) From: Robert Watson X-X-Sender: robert@fledge.watson.org To: Kip Macy In-Reply-To: <3c1674c90903071211y29ea8ee6u412352faf9cd108f@mail.gmail.com> Message-ID: References: <200903070118.n271IL5p034118@svn.freebsd.org> <9bbcef730903070240t2068f43em9b0017b10ba992e4@mail.gmail.com> <3c1674c90903071211y29ea8ee6u412352faf9cd108f@mail.gmail.com> User-Agent: Alpine 2.00 (BSF 1167 2008-08-23) MIME-Version: 1.0 Content-Type: MULTIPART/MIXED; BOUNDARY="621616949-1037175497-1236464350=:1340" Cc: src-committers@freebsd.org, Ivan Voras , svn-src-user@freebsd.org Subject: Re: svn commit: r189470 - in user/kmacy/releng_7_net_backport/sys: amd64/include arm/include dev/bce dev/mxge i386/include ia64/include powerpc/include sparc64/include sun4v/include X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 07 Mar 2009 22:19:11 -0000 This message is in MIME format. The first part should be readable text, while the remaining parts are likely unreadable without MIME-aware tools. --621616949-1037175497-1236464350=:1340 Content-Type: TEXT/PLAIN; charset=ISO-8859-1; format=flowed Content-Transfer-Encoding: 8BIT On Sat, 7 Mar 2009, Kip Macy wrote: > The names are what are used in Linux. There are a number of drivers in the > tree that share code with Linux that #define them in a platform dependent > way. As long as the definitions are consistently _KERNEL-only, I don't see that this is a problem; however, we should not expose them to user applications. Robert N M Watson Computer Laboratory University of Cambridge > > -Kip > > On Sat, Mar 7, 2009 at 2:40 AM, Ivan Voras wrote: >> Hi, >> >> Are these names taken from some standard? I'm not strongly arguing >> against them but naming something "mb" and such introduces great >> potential for conflicts with local code. (Not the actual choice of >> letters but that they are two and three-characters long identifiers). >> Maybe something like membar, membar_read, membar_write would be >> better? >> >> But I see this is merged from existing code so it's probably too late... >> >> >> 2009/3/7 Kip Macy : >>> Author: kmacy >>> Date: Sat Mar  7 01:18:21 2009 >>> New Revision: 189470 >>> URL: http://svn.freebsd.org/changeset/base/189470 >>> >>> Log: >>>   multiq step 1: >>>   - atomic_barriers: add atomic barriers to atomic.h on all architectures >>> >>> Modified: >>>  user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h >>>  user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h >>>  user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h >>>  user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c >>>  user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h >>>  user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h >>>  user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h >>>  user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h >>>  user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h >>>  user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/amd64/include/atomic.h Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -32,6 +32,10 @@ >>>  #error this file needs sys/cdefs.h as a prerequisite >>>  #endif >>> >>> +#define mb()   __asm__ __volatile__ ("mfence;": : :"memory") >>> +#define wmb()  __asm__ __volatile__ ("sfence;": : :"memory") >>> +#define rmb()  __asm__ __volatile__ ("lfence;": : :"memory") >>> + >>>  /* >>>  * Various simple operations on memory, each of which is atomic in the >>>  * presence of interrupts and multiple processors. >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h   Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/arm/include/atomic.h   Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -45,6 +45,10 @@ >>> >>>  #include >>> >>> +#define        mb() >>> +#define        wmb() >>> +#define        rmb() >>> + >>>  #ifndef I32_bit >>>  #define I32_bit (1 << 7)        /* IRQ disable */ >>>  #endif >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h    Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/dev/bce/if_bcereg.h    Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -557,6 +557,8 @@ >>> >>>  #endif /* BCE_DEBUG */ >>> >>> + >>> +#if __FreeBSD_version < 701101 >>>  #if defined(__i386__) || defined(__amd64__) >>>  #define mb()    __asm volatile("mfence" ::: "memory") >>>  #define wmb()   __asm volatile("sfence" ::: "memory") >>> @@ -566,6 +568,7 @@ >>>  #define rmb() >>>  #define wmb() >>>  #endif >>> +#endif >>> >>>  /****************************************************************************/ >>>  /* Device identification definitions.                                       */ >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c     Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge.c     Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -747,9 +747,9 @@ mxge_load_firmware_helper(mxge_softc_t * >>>                mxge_pio_copy(sc->sram + MXGE_FW_OFFSET + i, >>>                              inflate_buffer + i, >>>                              min(256U, (unsigned)(fw_len - i))); >>> -               mb(); >>> +               wmb(); >>>                dummy = *sc->sram; >>> -               mb(); >>> +               wmb(); >>>        } >>> >>>        *limit = fw_len; >>> @@ -782,7 +782,7 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en >>>        /* clear confirmation addr */ >>>        confirm = (volatile uint32_t *)sc->cmd; >>>        *confirm = 0; >>> -       mb(); >>> +       wmb(); >>> >>>        /* send an rdma command to the PCIe engine, and wait for the >>>           response in the confirmation address.  The firmware should >>> @@ -804,9 +804,9 @@ mxge_dummy_rdma(mxge_softc_t *sc, int en >>>        submit = (volatile char *)(sc->sram + MXGEFW_BOOT_DUMMY_RDMA); >>> >>>        mxge_pio_copy(submit, buf, 64); >>> -       mb(); >>> +       wmb(); >>>        DELAY(1000); >>> -       mb(); >>> +       wmb(); >>>        i = 0; >>>        while (*confirm != 0xffffffff && i < 20) { >>>                DELAY(1000); >>> @@ -844,7 +844,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t >>>        buf->response_addr.high = htobe32(dma_high); >>>        mtx_lock(&sc->cmd_mtx); >>>        response->result = 0xffffffff; >>> -       mb(); >>> +       wmb(); >>>        mxge_pio_copy((volatile void *)cmd_addr, buf, sizeof (*buf)); >>> >>>        /* wait up to 20ms */ >>> @@ -852,7 +852,7 @@ mxge_send_cmd(mxge_softc_t *sc, uint32_t >>>        for (sleep_total = 0; sleep_total <  20; sleep_total++) { >>>                bus_dmamap_sync(sc->cmd_dma.dmat, >>>                                sc->cmd_dma.map, BUS_DMASYNC_POSTREAD); >>> -               mb(); >>> +               wmb(); >>>                switch (be32toh(response->result)) { >>>                case 0: >>>                        data->data0 = be32toh(response->data); >>> @@ -980,7 +980,7 @@ mxge_load_firmware(mxge_softc_t *sc, int >>>        /* clear confirmation addr */ >>>        confirm = (volatile uint32_t *)sc->cmd; >>>        *confirm = 0; >>> -       mb(); >>> +       wmb(); >>>        /* send a reload command to the bootstrap MCP, and wait for the >>>           response in the confirmation address.  The firmware should >>>           write a -1 there to indicate it is alive and well >>> @@ -1005,9 +1005,9 @@ mxge_load_firmware(mxge_softc_t *sc, int >>> >>>        submit = (volatile char *)(sc->sram + MXGEFW_BOOT_HANDOFF); >>>        mxge_pio_copy(submit, buf, 64); >>> -       mb(); >>> +       wmb(); >>>        DELAY(1000); >>> -       mb(); >>> +       wmb(); >>>        i = 0; >>>        while (*confirm != 0xffffffff && i < 20) { >>>                DELAY(1000*10); >>> @@ -1668,7 +1668,7 @@ mxge_submit_req_backwards(mxge_tx_ring_t >>>                 idx = (starting_slot + cnt) & tx->mask; >>>                 mxge_pio_copy(&tx->lanai[idx], >>>                              &src[cnt], sizeof(*src)); >>> -                mb(); >>> +                wmb(); >>>         } >>>  } >>> >>> @@ -1694,14 +1694,14 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ >>> >>>        last_flags = src->flags; >>>        src->flags = 0; >>> -        mb(); >>> +        wmb(); >>>         dst = dstp = &tx->lanai[idx]; >>>         srcp = src; >>> >>>         if ((idx + cnt) < tx->mask) { >>>                 for (i = 0; i < (cnt - 1); i += 2) { >>>                         mxge_pio_copy(dstp, srcp, 2 * sizeof(*src)); >>> -                        mb(); /* force write every 32 bytes */ >>> +                        wmb(); /* force write every 32 bytes */ >>>                         srcp += 2; >>>                         dstp += 2; >>>                 } >>> @@ -1714,7 +1714,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ >>>         if (i < cnt) { >>>                 /* submit the first request */ >>>                 mxge_pio_copy(dstp, srcp, sizeof(*src)); >>> -                mb(); /* barrier before setting valid flag */ >>> +                wmb(); /* barrier before setting valid flag */ >>>         } >>> >>>         /* re-write the last 32-bits with the valid flags */ >>> @@ -1725,7 +1725,7 @@ mxge_submit_req(mxge_tx_ring_t *tx, mcp_ >>>         dst_ints+=3; >>>         *dst_ints =  *src_ints; >>>         tx->req += cnt; >>> -        mb(); >>> +        wmb(); >>>  } >>> >>>  #if IFCAP_TSO4 >>> @@ -2144,12 +2144,12 @@ mxge_submit_8rx(volatile mcp_kreq_ether_ >>>        low = src->addr_low; >>>        src->addr_low = 0xffffffff; >>>        mxge_pio_copy(dst, src, 4 * sizeof (*src)); >>> -       mb(); >>> +       wmb(); >>>        mxge_pio_copy(dst + 4, src + 4, 4 * sizeof (*src)); >>> -       mb(); >>> +       wmb(); >>>        src->addr_low = low; >>>        dst->addr_low = low; >>> -       mb(); >>> +       wmb(); >>>  } >>> >>>  static int >>> @@ -2736,7 +2736,7 @@ mxge_intr(void *arg) >>>                        send_done_count = be32toh(stats->send_done_count); >>>                } >>>                if (sc->legacy_irq && mxge_deassert_wait) >>> -                       mb(); >>> +                       wmb(); >>>        } while (*((volatile uint8_t *) &stats->valid)); >>> >>>        if (__predict_false(stats->stats_updated)) { >>> @@ -3385,7 +3385,7 @@ mxge_close(mxge_softc_t *sc) >>>        callout_stop(&sc->co_hdl); >>>        sc->ifp->if_drv_flags &= ~IFF_DRV_RUNNING; >>>        old_down_cnt = sc->down_cnt; >>> -       mb(); >>> +       wmb(); >>>        err = mxge_send_cmd(sc, MXGEFW_CMD_ETHERNET_DOWN, &cmd); >>>        if (err) { >>>                device_printf(sc->dev, "Couldn't bring down link\n"); >>> @@ -3394,7 +3394,7 @@ mxge_close(mxge_softc_t *sc) >>>                /* wait for down irq */ >>>                DELAY(10 * sc->intr_coal_delay); >>>        } >>> -       mb(); >>> +       wmb(); >>>        if (old_down_cnt == sc->down_cnt) { >>>                device_printf(sc->dev, "never got down irq\n"); >>>        } >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/dev/mxge/if_mxge_var.h Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -282,6 +282,8 @@ struct mxge_media_type >>>  /* implement our own memory barriers, since bus_space_barrier >>>    cannot handle write-combining regions */ >>> >>> +#if __FreeBSD_version < 701101 >>> + >>>  #if defined (__GNUC__) >>>   #if #cpu(i386) || defined __i386 || defined i386 || defined __i386__ || #cpu(x86_64) || defined __x86_64__ >>>     #define mb()  __asm__ __volatile__ ("sfence;": : :"memory") >>> @@ -296,6 +298,8 @@ struct mxge_media_type >>>   #error "unknown compiler" >>>  #endif >>> >>> +#endif >>> + >>>  static inline void >>>  mxge_pio_copy(volatile void *to_v, void *from_v, size_t size) >>>  { >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h  Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/i386/include/atomic.h  Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -32,6 +32,47 @@ >>>  #error this file needs sys/cdefs.h as a prerequisite >>>  #endif >>> >>> +#if defined(_KERNEL) >>> +#include >>> +#include >>> +#define        mb()    __asm __volatile(                       \ >>> +       "testl  %0,cpu_feature                          \n\ >>> +       je      2f                                      \n\ >>> +       mfence                                          \n\ >>> +1:                                                     \n\ >>> +       .section .text.offpath                          \n\ >>> +2:     lock                                            \n\ >>> +       addl    $0,cpu_feature                          \n\ >>> +       jmp     1b                                      \n\ >>> +       .text                                           \n\ >>> +"                                                      \ >>> +       : : "i"(CPUID_SSE2) : "memory") >>> +#define        wmb()   __asm __volatile(                       \ >>> +       "testl  %0,cpu_feature                          \n\ >>> +       je      2f                                      \n\ >>> +       sfence                                          \n\ >>> +1:                                                     \n\ >>> +       .section .text.offpath                          \n\ >>> +2:     lock                                            \n\ >>> +       addl    $0,cpu_feature                          \n\ >>> +       jmp     1b                                      \n\ >>> +       .text                                           \n\ >>> +"                                                      \ >>> +       : : "i"(CPUID_XMM) : "memory") >>> +#define        rmb()   __asm __volatile(                       \ >>> +       "testl  %0,cpu_feature                          \n\ >>> +       je      2f                                      \n\ >>> +       lfence                                          \n\ >>> +1:                                                     \n\ >>> +       .section .text.offpath                          \n\ >>> +2:     lock                                            \n\ >>> +       addl    $0,cpu_feature                          \n\ >>> +       jmp     1b                                      \n\ >>> +       .text                                           \n\ >>> +"                                                      \ >>> +       : : "i"(CPUID_SSE2) : "memory") >>> +#endif >>> + >>>  /* >>>  * Various simple operations on memory, each of which is atomic in the >>>  * presence of interrupts and multiple processors. >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h  Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/ia64/include/atomic.h  Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -29,6 +29,10 @@ >>>  #ifndef _MACHINE_ATOMIC_H_ >>>  #define        _MACHINE_ATOMIC_H_ >>> >>> +#define        mb() >>> +#define        wmb() >>> +#define        rmb() >>> + >>>  /* >>>  * Various simple arithmetic on memory which is atomic in the presence >>>  * of interrupts and SMP safe. >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h       Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/powerpc/include/atomic.h       Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -39,6 +39,10 @@ >>>  #define        __ATOMIC_BARRIER                                        \ >>>     __asm __volatile("sync" : : : "memory") >>> >>> +#define mb()   __ATOMIC_BARRIER >>> +#define        wmb()   mb() >>> +#define        rmb()   mb() >>> + >>>  /* >>>  * atomic_add(p, v) >>>  * { *p += v; } >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h       Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/sparc64/include/atomic.h       Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -40,6 +40,10 @@ >>>  #define        __ASI_ATOMIC    ASI_P >>>  #endif >>> >>> +#define mb()   __asm__ __volatile__ ("membar #MemIssue": : :"memory") >>> +#define wmb()  mb() >>> +#define rmb()  mb() >>> + >>>  /* >>>  * Various simple arithmetic on memory which is atomic in the presence >>>  * of interrupts and multiple processors.  See atomic(9) for details. >>> >>> Modified: user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h >>> ============================================================================== >>> --- user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Sat Mar  7 01:17:13 2009        (r189469) >>> +++ user/kmacy/releng_7_net_backport/sys/sun4v/include/atomic.h Sat Mar  7 01:18:21 2009        (r189470) >>> @@ -33,6 +33,10 @@ >>> >>>  #include >>> >>> +#define mb()   __asm__ __volatile__ ("membar #MemIssue": : :"memory") >>> +#define wmb()  mb() >>> +#define rmb()  mb() >>> + >>>  /* Userland needs different ASI's. */ >>>  #ifdef _KERNEL >>>  #define        __ASI_ATOMIC    ASI_N >>> >> > --621616949-1037175497-1236464350=:1340--