From owner-svn-src-all@freebsd.org Wed Jan 22 05:53:18 2020 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id BBEAF22DC9F; Wed, 22 Jan 2020 05:53:18 +0000 (UTC) (envelope-from glebius@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 482ZML2r5Mz4Zxt; Wed, 22 Jan 2020 05:53:18 +0000 (UTC) (envelope-from glebius@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 5CA6E20DB3; Wed, 22 Jan 2020 05:53:18 +0000 (UTC) (envelope-from glebius@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 00M5rIbb024734; Wed, 22 Jan 2020 05:53:18 GMT (envelope-from glebius@FreeBSD.org) Received: (from glebius@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 00M5rHG4024077; Wed, 22 Jan 2020 05:53:17 GMT (envelope-from glebius@FreeBSD.org) Message-Id: <202001220553.00M5rHG4024077@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: glebius set sender to glebius@FreeBSD.org using -f From: Gleb Smirnoff Date: Wed, 22 Jan 2020 05:53:17 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r356975 - in head/sys/netinet: . tcp_stacks X-SVN-Group: head X-SVN-Commit-Author: glebius X-SVN-Commit-Paths: in head/sys/netinet: . tcp_stacks X-SVN-Commit-Revision: 356975 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 22 Jan 2020 05:53:18 -0000 Author: glebius Date: Wed Jan 22 05:53:16 2020 New Revision: 356975 URL: https://svnweb.freebsd.org/changeset/base/356975 Log: Make tcp_output() require network epoch. Enter the epoch before calling into tcp_output() from those functions, that didn't do that before. This eliminates a bunch of epoch recursions in TCP. Modified: head/sys/netinet/tcp_output.c head/sys/netinet/tcp_stacks/bbr.c head/sys/netinet/tcp_stacks/rack.c head/sys/netinet/tcp_timer.c head/sys/netinet/tcp_usrreq.c head/sys/netinet/toecore.c Modified: head/sys/netinet/tcp_output.c ============================================================================== --- head/sys/netinet/tcp_output.c Wed Jan 22 05:51:22 2020 (r356974) +++ head/sys/netinet/tcp_output.c Wed Jan 22 05:53:16 2020 (r356975) @@ -193,7 +193,6 @@ cc_after_idle(struct tcpcb *tp) int tcp_output(struct tcpcb *tp) { - struct epoch_tracker et; struct socket *so = tp->t_inpcb->inp_socket; int32_t len; uint32_t recwin, sendwin; @@ -233,6 +232,7 @@ tcp_output(struct tcpcb *tp) const bool hw_tls = false; #endif + NET_EPOCH_ASSERT(); INP_WLOCK_ASSERT(tp->t_inpcb); #ifdef TCP_OFFLOAD @@ -1372,7 +1372,6 @@ send: * m->m_pkthdr.len should have been set before checksum calculation, * because in6_cksum() need it. */ - NET_EPOCH_ENTER(et); #ifdef INET6 if (isipv6) { /* @@ -1458,7 +1457,6 @@ send: mtu = tp->t_inpcb->inp_route.ro_rt->rt_mtu; } #endif /* INET */ - NET_EPOCH_EXIT(et); out: /* Modified: head/sys/netinet/tcp_stacks/bbr.c ============================================================================== --- head/sys/netinet/tcp_stacks/bbr.c Wed Jan 22 05:51:22 2020 (r356974) +++ head/sys/netinet/tcp_stacks/bbr.c Wed Jan 22 05:53:16 2020 (r356975) @@ -12091,7 +12091,6 @@ bbr_window_update_needed(struct tcpcb *tp, struct sock static int bbr_output_wtime(struct tcpcb *tp, const struct timeval *tv) { - struct epoch_tracker et; struct socket *so; int32_t len; uint32_t cts; @@ -13938,7 +13937,6 @@ send: * m->m_pkthdr.len should have been set before cksum calcuration, * because in6_cksum() need it. */ - NET_EPOCH_ENTER(et); #ifdef INET6 if (isipv6) { /* @@ -14016,7 +14014,6 @@ send: mtu = inp->inp_route.ro_rt->rt_mtu; } #endif /* INET */ - NET_EPOCH_EXIT(et); out: if (lgb) { @@ -14464,6 +14461,8 @@ bbr_output(struct tcpcb *tp) int32_t ret; struct timeval tv; struct tcp_bbr *bbr; + + NET_EPOCH_ASSERT(); bbr = (struct tcp_bbr *)tp->t_fb_ptr; INP_WLOCK_ASSERT(tp->t_inpcb); Modified: head/sys/netinet/tcp_stacks/rack.c ============================================================================== --- head/sys/netinet/tcp_stacks/rack.c Wed Jan 22 05:51:22 2020 (r356974) +++ head/sys/netinet/tcp_stacks/rack.c Wed Jan 22 05:53:16 2020 (r356975) @@ -8091,7 +8091,6 @@ old_method: static int rack_output(struct tcpcb *tp) { - struct epoch_tracker et; struct socket *so; uint32_t recwin, sendwin; uint32_t sb_offset; @@ -8155,8 +8154,10 @@ rack_output(struct tcpcb *tp) #ifdef KERN_TLS hw_tls = (so->so_snd.sb_flags & SB_TLS_IFNET) != 0; #endif - + + NET_EPOCH_ASSERT(); INP_WLOCK_ASSERT(inp); + #ifdef TCP_OFFLOAD if (tp->t_flags & TF_TOE) return (tcp_offload_output(tp)); @@ -9734,7 +9735,6 @@ send: * m->m_pkthdr.len should have been set before cksum calcuration, * because in6_cksum() need it. */ - NET_EPOCH_ENTER(et); #ifdef INET6 if (isipv6) { /* @@ -9812,7 +9812,6 @@ send: mtu = inp->inp_route.ro_rt->rt_mtu; } #endif /* INET */ - NET_EPOCH_EXIT(et); out: if (lgb) { Modified: head/sys/netinet/tcp_timer.c ============================================================================== --- head/sys/netinet/tcp_timer.c Wed Jan 22 05:51:22 2020 (r356974) +++ head/sys/netinet/tcp_timer.c Wed Jan 22 05:53:16 2020 (r356975) @@ -251,6 +251,7 @@ int tcp_totbackoff = 2559; /* sum of tcp_backoff[] */ void tcp_timer_delack(void *xtp) { + struct epoch_tracker et; struct tcpcb *tp = xtp; struct inpcb *inp; CURVNET_SET(tp->t_vnet); @@ -272,8 +273,10 @@ tcp_timer_delack(void *xtp) } tp->t_flags |= TF_ACKNOW; TCPSTAT_INC(tcps_delack); + NET_EPOCH_ENTER(et); (void) tp->t_fb->tfb_tcp_output(tp); INP_WUNLOCK(inp); + NET_EPOCH_EXIT(et); CURVNET_RESTORE(); } @@ -570,7 +573,9 @@ tcp_timer_persist(void *xtp) } tcp_setpersist(tp); tp->t_flags |= TF_FORCEDATA; + NET_EPOCH_ENTER(et); (void) tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); tp->t_flags &= ~TF_FORCEDATA; #ifdef TCPDEBUG @@ -824,9 +829,9 @@ tcp_timer_rexmt(void * xtp) tp->t_rtttime = 0; cc_cong_signal(tp, NULL, CC_RTO); - + NET_EPOCH_ENTER(et); (void) tp->t_fb->tfb_tcp_output(tp); - + NET_EPOCH_EXIT(et); #ifdef TCPDEBUG if (tp != NULL && (tp->t_inpcb->inp_socket->so_options & SO_DEBUG)) tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0, Modified: head/sys/netinet/tcp_usrreq.c ============================================================================== --- head/sys/netinet/tcp_usrreq.c Wed Jan 22 05:51:22 2020 (r356974) +++ head/sys/netinet/tcp_usrreq.c Wed Jan 22 05:53:16 2020 (r356975) @@ -531,6 +531,7 @@ out: static int tcp_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) { + struct epoch_tracker et; int error = 0; struct inpcb *inp; struct tcpcb *tp = NULL; @@ -571,7 +572,9 @@ tcp_usr_connect(struct socket *so, struct sockaddr *na goto out; #endif tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); out: TCPDEBUG2(PRU_CONNECT); TCP_PROBE2(debug__user, tp, PRU_CONNECT); @@ -584,6 +587,7 @@ out: static int tcp6_usr_connect(struct socket *so, struct sockaddr *nam, struct thread *td) { + struct epoch_tracker et; int error = 0; struct inpcb *inp; struct tcpcb *tp = NULL; @@ -654,7 +658,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *n (error = tcp_offload_connect(so, nam)) == 0) goto out; #endif + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); goto out; } else { if ((inp->inp_vflag & INP_IPV6) == 0) { @@ -677,8 +683,9 @@ tcp6_usr_connect(struct socket *so, struct sockaddr *n goto out; #endif tcp_timer_activate(tp, TT_KEEP, TP_KEEPINIT(tp)); + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); - + NET_EPOCH_EXIT(et); out: /* * If the implicit bind in the connect call fails, restore @@ -882,6 +889,7 @@ out: static int tcp_usr_rcvd(struct socket *so, int flags) { + struct epoch_tracker et; struct inpcb *inp; struct tcpcb *tp = NULL; int error = 0; @@ -911,8 +919,9 @@ tcp_usr_rcvd(struct socket *so, int flags) tcp_offload_rcvd(tp); else #endif + NET_EPOCH_ENTER(et); tp->t_fb->tfb_tcp_output(tp); - + NET_EPOCH_EXIT(et); out: TCPDEBUG2(PRU_RCVD); TCP_PROBE2(debug__user, tp, PRU_RCVD); @@ -953,8 +962,7 @@ tcp_usr_send(struct socket *so, int flags, struct mbuf * We require the pcbinfo "read lock" if we will close the socket * as part of this call. */ - if (flags & PRUS_EOF) - NET_EPOCH_ENTER(et); + NET_EPOCH_ENTER(et); inp = sotoinpcb(so); KASSERT(inp != NULL, ("tcp_usr_send: inp == NULL")); INP_WLOCK(inp); @@ -1240,14 +1248,14 @@ out: TCP_PROBE2(debug__user, tp, (flags & PRUS_OOB) ? PRU_SENDOOB : ((flags & PRUS_EOF) ? PRU_SEND_EOF : PRU_SEND)); INP_WUNLOCK(inp); - if (flags & PRUS_EOF) - NET_EPOCH_EXIT(et); + NET_EPOCH_EXIT(et); return (error); } static int tcp_usr_ready(struct socket *so, struct mbuf *m, int count) { + struct epoch_tracker et; struct inpcb *inp; struct tcpcb *tp; int error; @@ -1264,8 +1272,11 @@ tcp_usr_ready(struct socket *so, struct mbuf *m, int c SOCKBUF_LOCK(&so->so_snd); error = sbready(&so->so_snd, m, count); SOCKBUF_UNLOCK(&so->so_snd); - if (error == 0) + if (error == 0) { + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); + } INP_WUNLOCK(inp); return (error); @@ -1921,8 +1932,13 @@ unlock_and_done: tp->t_flags |= TF_NOPUSH; else if (tp->t_flags & TF_NOPUSH) { tp->t_flags &= ~TF_NOPUSH; - if (TCPS_HAVEESTABLISHED(tp->t_state)) + if (TCPS_HAVEESTABLISHED(tp->t_state)) { + struct epoch_tracker et; + + NET_EPOCH_ENTER(et); error = tp->t_fb->tfb_tcp_output(tp); + NET_EPOCH_EXIT(et); + } } goto unlock_and_done; Modified: head/sys/netinet/toecore.c ============================================================================== --- head/sys/netinet/toecore.c Wed Jan 22 05:51:22 2020 (r356974) +++ head/sys/netinet/toecore.c Wed Jan 22 05:53:16 2020 (r356975) @@ -503,6 +503,7 @@ void toe_connect_failed(struct toedev *tod, struct inpcb *inp, int err) { + NET_EPOCH_ASSERT(); INP_WLOCK_ASSERT(inp); if (!(inp->inp_flags & INP_DROPPED)) { @@ -527,7 +528,6 @@ toe_connect_failed(struct toedev *tod, struct inpcb *i (void) tp->t_fb->tfb_tcp_output(tp); } else { - NET_EPOCH_ASSERT(); tp = tcp_drop(tp, err); if (tp == NULL) INP_WLOCK(inp); /* re-acquire */