Skip site navigation (1)Skip section navigation (2)
Date:      Sat, 19 Dec 2020 22:04:46 +0000 (UTC)
From:      Andrew Gallatin <gallatin@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r368819 - in head: share/man/man4 sys/netinet sys/netinet6
Message-ID:  <202012192204.0BJM4kk5092379@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: gallatin
Date: Sat Dec 19 22:04:46 2020
New Revision: 368819
URL: https://svnweb.freebsd.org/changeset/base/368819

Log:
  Filter TCP connections to SO_REUSEPORT_LB listen sockets by NUMA domain
  
  In order to efficiently serve web traffic on a NUMA
  machine, one must avoid as many NUMA domain crossings as
  possible. With SO_REUSEPORT_LB, a number of workers can share a
  listen socket. However, even if a worker sets affinity to a core
  or set of cores on a NUMA domain, it will receive connections
  associated with all NUMA domains in the system. This will lead to
  cross-domain traffic when the server writes to the socket or
  calls sendfile(), and memory is allocated on the server's local
  NUMA node, but transmitted on the NUMA node associated with the
  TCP connection. Similarly, when the server reads from the socket,
  he will likely be reading memory allocated on the NUMA domain
  associated with the TCP connection.
  
  This change provides a new socket ioctl, TCP_REUSPORT_LB_NUMA. A
  server can now tell the kernel to filter traffic so that only
  incoming connections associated with the desired NUMA domain are
  given to the server. (Of course, in the case where there are no
  servers sharing the listen socket on some domain, then as a
  fallback, traffic will be hashed as normal to all servers sharing
  the listen socket regardless of domain). This allows a server to
  deal only with traffic that is local to its NUMA domain, and
  avoids cross-domain traffic in most cases.
  
  This patch, and a corresponding small patch to nginx to use
  TCP_REUSPORT_LB_NUMA allows us to serve 190Gb/s of kTLS encrypted
  https media content from dual-socket Xeons with only 13% (as
  measured by pcm.x) cross domain traffic on the memory controller.
  
  Reviewed by:	jhb, bz (earlier version), bcr (man page)
  Tested by: gonzo
  Sponsored by:	Netfix
  Differential Revision:	https://reviews.freebsd.org/D21636

Modified:
  head/share/man/man4/tcp.4
  head/sys/netinet/in_pcb.c
  head/sys/netinet/in_pcb.h
  head/sys/netinet/tcp.h
  head/sys/netinet/tcp_usrreq.c
  head/sys/netinet6/in6_pcb.c
  head/sys/netinet6/in6_pcb.h

Modified: head/share/man/man4/tcp.4
==============================================================================
--- head/share/man/man4/tcp.4	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/share/man/man4/tcp.4	Sat Dec 19 22:04:46 2020	(r368819)
@@ -34,7 +34,7 @@
 .\"     From: @(#)tcp.4	8.1 (Berkeley) 6/5/93
 .\" $FreeBSD$
 .\"
-.Dd November 25, 2020
+.Dd December 19, 2020
 .Dt TCP 4
 .Os
 .Sh NAME
@@ -314,6 +314,21 @@ Enable in-kernel TLS for data read from this socket.
 See
 .Xr ktls 4
 for more details.
+.It Dv TCP_REUSPORT_LB_NUMA
+Changes NUMA affinity filtering for an established TCP listen
+socket.
+This option takes a single integer argument which specifies
+the NUMA domain to filter on for this listen socket.
+The argument can also have the follwing special values:
+.Bl -tag -width "Dv TCP_REUSPORT_LB_NUMA"
+.It Dv TCP_REUSPORT_LB_NUMA_NODOM
+Remove NUMA filtering for this listen socket.
+.It Dv TCP_REUSPORT_LB_NUMA_CURDOM
+Filter traffic associated with the domain where the calling thread is
+currently executing.
+This is typically used after a process or thread inherits a listen
+socket from its parent, and sets its CPU affinity to a particular core.
+.El
 .El
 .Pp
 The option level for the

Modified: head/sys/netinet/in_pcb.c
==============================================================================
--- head/sys/netinet/in_pcb.c	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/sys/netinet/in_pcb.c	Sat Dec 19 22:04:46 2020	(r368819)
@@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$");
 #endif
 
 #include <vm/uma.h>
+#include <vm/vm.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
@@ -150,7 +151,8 @@ static void	in_pcbremlists(struct inpcb *inp);
 static struct inpcb	*in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 			    struct in_addr faddr, u_int fport_arg,
 			    struct in_addr laddr, u_int lport_arg,
-			    int lookupflags, struct ifnet *ifp);
+			    int lookupflags, struct ifnet *ifp,
+			    uint8_t numa_domain);
 
 #define RANGECHK(var, min, max) \
 	if ((var) < (min)) { (var) = (min); } \
@@ -248,7 +250,8 @@ SYSCTL_COUNTER_U64(_net_inet_ip_rl, OID_AUTO, set_ok, 
 
 static struct inpcblbgroup *
 in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag,
-    uint16_t port, const union in_dependaddr *addr, int size)
+    uint16_t port, const union in_dependaddr *addr, int size,
+    uint8_t numa_domain)
 {
 	struct inpcblbgroup *grp;
 	size_t bytes;
@@ -259,6 +262,7 @@ in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_ch
 		return (NULL);
 	grp->il_vflag = vflag;
 	grp->il_lport = port;
+	grp->il_numa_domain = numa_domain;
 	grp->il_dependladdr = *addr;
 	grp->il_inpsiz = size;
 	CK_LIST_INSERT_HEAD(hdr, grp, il_list);
@@ -290,7 +294,8 @@ in_pcblbgroup_resize(struct inpcblbgrouphead *hdr,
 	int i;
 
 	grp = in_pcblbgroup_alloc(hdr, old_grp->il_vflag,
-	    old_grp->il_lport, &old_grp->il_dependladdr, size);
+	    old_grp->il_lport, &old_grp->il_dependladdr, size,
+	    old_grp->il_numa_domain);
 	if (grp == NULL)
 		return (NULL);
 
@@ -333,7 +338,7 @@ in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, st
  * Add PCB to load balance group for SO_REUSEPORT_LB option.
  */
 static int
-in_pcbinslbgrouphash(struct inpcb *inp)
+in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
 {
 	const static struct timeval interval = { 60, 0 };
 	static struct timeval lastprint;
@@ -369,6 +374,7 @@ in_pcbinslbgrouphash(struct inpcb *inp)
 	CK_LIST_FOREACH(grp, hdr, il_list) {
 		if (grp->il_vflag == inp->inp_vflag &&
 		    grp->il_lport == inp->inp_lport &&
+		    grp->il_numa_domain == numa_domain &&
 		    memcmp(&grp->il_dependladdr,
 		    &inp->inp_inc.inc_ie.ie_dependladdr,
 		    sizeof(grp->il_dependladdr)) == 0)
@@ -378,7 +384,7 @@ in_pcbinslbgrouphash(struct inpcb *inp)
 		/* Create new load balance group. */
 		grp = in_pcblbgroup_alloc(hdr, inp->inp_vflag,
 		    inp->inp_lport, &inp->inp_inc.inc_ie.ie_dependladdr,
-		    INPCBLBGROUP_SIZMIN);
+		    INPCBLBGROUP_SIZMIN, numa_domain);
 		if (grp == NULL)
 			return (ENOBUFS);
 	} else if (grp->il_inpcnt == grp->il_inpsiz) {
@@ -439,6 +445,57 @@ in_pcbremlbgrouphash(struct inpcb *inp)
 	}
 }
 
+int
+in_pcblbgroup_numa(struct inpcb *inp, int arg)
+{
+	struct inpcbinfo *pcbinfo;
+	struct inpcblbgrouphead *hdr;
+	struct inpcblbgroup *grp;
+	int err, i;
+	uint8_t numa_domain;
+
+	switch (arg) {
+	case TCP_REUSPORT_LB_NUMA_NODOM:
+		numa_domain = M_NODOM;
+		break;
+	case TCP_REUSPORT_LB_NUMA_CURDOM:
+		numa_domain = PCPU_GET(domain);
+		break;
+	default:
+		if (arg < 0 || arg >= vm_ndomains)
+			return (EINVAL);
+		numa_domain = arg;
+	}
+
+	err = 0;
+	pcbinfo = inp->inp_pcbinfo;
+	INP_WLOCK_ASSERT(inp);
+	INP_HASH_WLOCK(pcbinfo);
+	hdr = &pcbinfo->ipi_lbgrouphashbase[
+	    INP_PCBPORTHASH(inp->inp_lport, pcbinfo->ipi_lbgrouphashmask)];
+	CK_LIST_FOREACH(grp, hdr, il_list) {
+		for (i = 0; i < grp->il_inpcnt; ++i) {
+			if (grp->il_inp[i] != inp)
+				continue;
+
+			if (grp->il_numa_domain == numa_domain) {
+				goto abort_with_hash_wlock;
+			}
+
+			/* Remove it from the old group. */
+			in_pcbremlbgrouphash(inp);
+
+			/* Add it to the new group based on numa domain. */
+			in_pcbinslbgrouphash(inp, numa_domain);
+			goto abort_with_hash_wlock;
+		}
+	}
+	err = ENOENT;
+abort_with_hash_wlock:
+	INP_HASH_WUNLOCK(pcbinfo);
+	return (err);
+}
+
 /*
  * Different protocols initialize their inpcbs differently - giving
  * different name to the lock.  But they all are disposed the same.
@@ -731,14 +788,14 @@ in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *
 			if (lsa->sa_family == AF_INET) {
 				tmpinp = in_pcblookup_hash_locked(pcbinfo,
 				    faddr, fport, laddr, lport, lookupflags,
-				    NULL);
+				    NULL, M_NODOM);
 			}
 #endif
 #ifdef INET6
 			if (lsa->sa_family == AF_INET6) {
 				tmpinp = in6_pcblookup_hash_locked(pcbinfo,
 				    faddr6, fport, laddr6, lport, lookupflags,
-				    NULL);
+				    NULL, M_NODOM);
 			}
 #endif
 		} else {
@@ -1399,9 +1456,10 @@ in_pcbconnect_setup(struct inpcb *inp, struct sockaddr
 		if (error)
 			return (error);
 	}
+
 	if (lport != 0) {
 		oinp = in_pcblookup_hash_locked(inp->inp_pcbinfo, faddr,
-		    fport, laddr, lport, 0, NULL);
+		    fport, laddr, lport, 0, NULL, M_NODOM);
 		if (oinp != NULL) {
 			if (oinpp != NULL)
 				*oinpp = oinp;
@@ -2019,9 +2077,9 @@ in_pcblookup_local(struct inpcbinfo *pcbinfo, struct i
 static struct inpcb *
 in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
     const struct in_addr *laddr, uint16_t lport, const struct in_addr *faddr,
-    uint16_t fport, int lookupflags)
+    uint16_t fport, int lookupflags, int numa_domain)
 {
-	struct inpcb *local_wild;
+	struct inpcb *local_wild, *numa_wild;
 	const struct inpcblbgrouphead *hdr;
 	struct inpcblbgroup *grp;
 	uint32_t idx;
@@ -2041,6 +2099,7 @@ in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
 	 * - Load balanced group does not contain IPv4 mapped INET6 wild sockets
 	 */
 	local_wild = NULL;
+	numa_wild = NULL;
 	CK_LIST_FOREACH(grp, hdr, il_list) {
 #ifdef INET6
 		if (!(grp->il_vflag & INP_IPV4))
@@ -2051,12 +2110,24 @@ in_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
 
 		idx = INP_PCBLBGROUP_PKTHASH(faddr->s_addr, lport, fport) %
 		    grp->il_inpcnt;
-		if (grp->il_laddr.s_addr == laddr->s_addr)
-			return (grp->il_inp[idx]);
+		if (grp->il_laddr.s_addr == laddr->s_addr) {
+			if (numa_domain == M_NODOM ||
+			    grp->il_numa_domain == numa_domain) {
+				return (grp->il_inp[idx]);
+			} else {
+				numa_wild = grp->il_inp[idx];
+			}
+		}
 		if (grp->il_laddr.s_addr == INADDR_ANY &&
-		    (lookupflags & INPLOOKUP_WILDCARD) != 0)
+		    (lookupflags & INPLOOKUP_WILDCARD) != 0 &&
+		    (local_wild == NULL || numa_domain == M_NODOM ||
+			grp->il_numa_domain == numa_domain)) {
 			local_wild = grp->il_inp[idx];
+		}
 	}
+	if (numa_wild != NULL)
+		return (numa_wild);
+
 	return (local_wild);
 }
 
@@ -2303,7 +2374,7 @@ found:
 static struct inpcb *
 in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in_addr faddr,
     u_int fport_arg, struct in_addr laddr, u_int lport_arg, int lookupflags,
-    struct ifnet *ifp)
+    struct ifnet *ifp, uint8_t numa_domain)
 {
 	struct inpcbhead *head;
 	struct inpcb *inp, *tmpinp;
@@ -2348,7 +2419,7 @@ in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, st
 	 */
 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
 		inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
-		    fport, lookupflags);
+		    fport, lookupflags, numa_domain);
 		if (inp != NULL)
 			return (inp);
 	}
@@ -2435,12 +2506,13 @@ in_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, st
 static struct inpcb *
 in_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in_addr faddr,
     u_int fport, struct in_addr laddr, u_int lport, int lookupflags,
-    struct ifnet *ifp)
+    struct ifnet *ifp, uint8_t numa_domain)
 {
 	struct inpcb *inp;
 
 	inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
-	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
+	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp,
+	    numa_domain);
 	if (inp != NULL) {
 		if (lookupflags & INPLOOKUP_WLOCKPCB) {
 			INP_WLOCK(inp);
@@ -2507,7 +2579,7 @@ in_pcblookup(struct inpcbinfo *pcbinfo, struct in_addr
 	}
 #endif
 	return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
-	    lookupflags, ifp));
+	    lookupflags, ifp, M_NODOM));
 }
 
 struct inpcb *
@@ -2549,7 +2621,7 @@ in_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct in
 	}
 #endif
 	return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
-	    lookupflags, ifp));
+	    lookupflags, ifp, m->m_pkthdr.numa_domain));
 }
 #endif /* INET */
 
@@ -2591,7 +2663,7 @@ in_pcbinshash_internal(struct inpcb *inp, struct mbuf 
 	 */
 	so_options = inp_so_options(inp);
 	if (so_options & SO_REUSEPORT_LB) {
-		int ret = in_pcbinslbgrouphash(inp);
+		int ret = in_pcbinslbgrouphash(inp, M_NODOM);
 		if (ret) {
 			/* pcb lb group malloc fail (ret=ENOBUFS). */
 			return (ret);

Modified: head/sys/netinet/in_pcb.h
==============================================================================
--- head/sys/netinet/in_pcb.h	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/sys/netinet/in_pcb.h	Sat Dec 19 22:04:46 2020	(r368819)
@@ -565,7 +565,7 @@ struct inpcblbgroup {
 	struct epoch_context il_epoch_ctx;
 	uint16_t	il_lport;			/* (c) */
 	u_char		il_vflag;			/* (c) */
-	u_char		il_pad;
+	u_int8_t		il_numa_domain;
 	uint32_t	il_pad2;
 	union in_dependaddr il_dependladdr;		/* (c) */
 #define	il_laddr	il_dependladdr.id46_addr.ia46_addr4
@@ -852,6 +852,7 @@ int	in_pcbinshash(struct inpcb *);
 int	in_pcbinshash_mbuf(struct inpcb *, struct mbuf *);
 int	in_pcbladdr(struct inpcb *, struct in_addr *, struct in_addr *,
 	    struct ucred *);
+int	in_pcblbgroup_numa(struct inpcb *, int arg);
 struct inpcb *
 	in_pcblookup_local(struct inpcbinfo *,
 	    struct in_addr, u_short, int, struct ucred *);

Modified: head/sys/netinet/tcp.h
==============================================================================
--- head/sys/netinet/tcp.h	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/sys/netinet/tcp.h	Sat Dec 19 22:04:46 2020	(r368819)
@@ -196,6 +196,7 @@ struct tcphdr {
 #define	TCP_PCAP_IN	4096	/* number of input packets to keep */
 #define TCP_FUNCTION_BLK 8192	/* Set the tcp function pointers to the specified stack */
 /* Options for Rack and BBR */
+#define	TCP_REUSPORT_LB_NUMA   1026	/* set listen socket numa domain */
 #define TCP_RACK_MBUF_QUEUE   1050 /* Do we allow mbuf queuing if supported */
 #define TCP_RACK_PROP	      1051 /* RACK proportional rate reduction (bool) */
 #define TCP_RACK_TLP_REDUCE   1052 /* RACK TLP cwnd reduction (bool) */
@@ -405,5 +406,8 @@ struct tcp_function_set {
 #define	VOI_TCP_CALCFRWINDIFF	7 /* Congestion avoidance LCWIN - FRWIN */
 #define	VOI_TCP_GPUT_ND		8 /* Goodput normalised delta */
 #define	VOI_TCP_ACKLEN		9 /* Average ACKed bytes per ACK */
+
+#define TCP_REUSPORT_LB_NUMA_NODOM	(-2) /* remove numa binding */
+#define TCP_REUSPORT_LB_NUMA_CURDOM	(-1) /* bind to current domain */
 
 #endif /* !_NETINET_TCP_H_ */

Modified: head/sys/netinet/tcp_usrreq.c
==============================================================================
--- head/sys/netinet/tcp_usrreq.c	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/sys/netinet/tcp_usrreq.c	Sat Dec 19 22:04:46 2020	(r368819)
@@ -2143,6 +2143,16 @@ unlock_and_done:
 			INP_WUNLOCK(inp);
 			break;
 
+		case TCP_REUSPORT_LB_NUMA:
+			INP_WUNLOCK(inp);
+			error = sooptcopyin(sopt, &optval, sizeof(optval),
+			    sizeof(optval));
+			INP_WLOCK_RECHECK(inp);
+			if (!error)
+				error = in_pcblbgroup_numa(inp, optval);
+			INP_WUNLOCK(inp);
+			break;
+
 #ifdef KERN_TLS
 		case TCP_TXTLS_ENABLE:
 			INP_WUNLOCK(inp);

Modified: head/sys/netinet6/in6_pcb.c
==============================================================================
--- head/sys/netinet6/in6_pcb.c	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/sys/netinet6/in6_pcb.c	Sat Dec 19 22:04:46 2020	(r368819)
@@ -446,7 +446,7 @@ in6_pcbconnect_mbuf(struct inpcb *inp, struct sockaddr
 			       sin6->sin6_port,
 			      IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)
 			      ? &laddr6.sin6_addr : &inp->in6p_laddr,
-			      inp->inp_lport, 0, NULL) != NULL) {
+			      inp->inp_lport, 0, NULL, M_NODOM) != NULL) {
 		return (EADDRINUSE);
 	}
 	if (IN6_IS_ADDR_UNSPECIFIED(&inp->in6p_laddr)) {
@@ -903,9 +903,9 @@ in6_rtchange(struct inpcb *inp, int errno __unused)
 static struct inpcb *
 in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
     const struct in6_addr *laddr, uint16_t lport, const struct in6_addr *faddr,
-    uint16_t fport, int lookupflags)
+    uint16_t fport, int lookupflags, uint8_t numa_domain)
 {
-	struct inpcb *local_wild;
+	struct inpcb *local_wild, *numa_wild;
 	const struct inpcblbgrouphead *hdr;
 	struct inpcblbgroup *grp;
 	uint32_t idx;
@@ -925,6 +925,7 @@ in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
 	 * - Load balanced does not contain IPv4 mapped INET6 wild sockets.
 	 */
 	local_wild = NULL;
+	numa_wild = NULL;
 	CK_LIST_FOREACH(grp, hdr, il_list) {
 #ifdef INET
 		if (!(grp->il_vflag & INP_IPV6))
@@ -935,12 +936,23 @@ in6_pcblookup_lbgroup(const struct inpcbinfo *pcbinfo,
 
 		idx = INP_PCBLBGROUP_PKTHASH(INP6_PCBHASHKEY(faddr), lport,
 		    fport) % grp->il_inpcnt;
-		if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr))
-			return (grp->il_inp[idx]);
+		if (IN6_ARE_ADDR_EQUAL(&grp->il6_laddr, laddr)) {
+			if (numa_domain == M_NODOM ||
+			    grp->il_numa_domain == numa_domain) {
+				return (grp->il_inp[idx]);
+			}
+			else
+				numa_wild = grp->il_inp[idx];
+		}
 		if (IN6_IS_ADDR_UNSPECIFIED(&grp->il6_laddr) &&
-		    (lookupflags & INPLOOKUP_WILDCARD) != 0)
+		    (lookupflags & INPLOOKUP_WILDCARD) != 0 &&
+		    (local_wild == NULL || numa_domain == M_NODOM ||
+			grp->il_numa_domain == numa_domain)) {
 			local_wild = grp->il_inp[idx];
+		}
 	}
+	if (numa_wild != NULL)
+		return (numa_wild);
 	return (local_wild);
 }
 
@@ -1151,7 +1163,7 @@ found:
 struct inpcb *
 in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
     u_int fport_arg, struct in6_addr *laddr, u_int lport_arg,
-    int lookupflags, struct ifnet *ifp)
+    int lookupflags, struct ifnet *ifp, uint8_t numa_domain)
 {
 	struct inpcbhead *head;
 	struct inpcb *inp, *tmpinp;
@@ -1195,7 +1207,7 @@ in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, s
 	 */
 	if ((lookupflags & INPLOOKUP_WILDCARD) != 0) {
 		inp = in6_pcblookup_lbgroup(pcbinfo, laddr, lport, faddr,
-		    fport, lookupflags);
+		    fport, lookupflags, numa_domain);
 		if (inp != NULL)
 			return (inp);
 	}
@@ -1273,12 +1285,13 @@ in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo, s
 static struct inpcb *
 in6_pcblookup_hash(struct inpcbinfo *pcbinfo, struct in6_addr *faddr,
     u_int fport, struct in6_addr *laddr, u_int lport, int lookupflags,
-    struct ifnet *ifp)
+    struct ifnet *ifp, uint8_t numa_domain)
 {
 	struct inpcb *inp;
 
 	inp = in6_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
-	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp);
+	    (lookupflags & ~(INPLOOKUP_RLOCKPCB | INPLOOKUP_WLOCKPCB)), ifp,
+	    numa_domain);
 	if (inp != NULL) {
 		if (lookupflags & INPLOOKUP_WLOCKPCB) {
 			INP_WLOCK(inp);
@@ -1344,7 +1357,7 @@ in6_pcblookup(struct inpcbinfo *pcbinfo, struct in6_ad
 	}
 #endif
 	return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
-	    lookupflags, ifp));
+	    lookupflags, ifp, M_NODOM));
 }
 
 struct inpcb *
@@ -1386,7 +1399,7 @@ in6_pcblookup_mbuf(struct inpcbinfo *pcbinfo, struct i
 	}
 #endif
 	return (in6_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
-	    lookupflags, ifp));
+	    lookupflags, ifp, m->m_pkthdr.numa_domain));
 }
 
 void

Modified: head/sys/netinet6/in6_pcb.h
==============================================================================
--- head/sys/netinet6/in6_pcb.h	Sat Dec 19 21:46:09 2020	(r368818)
+++ head/sys/netinet6/in6_pcb.h	Sat Dec 19 22:04:46 2020	(r368819)
@@ -95,7 +95,7 @@ struct	inpcb *
 struct inpcb *
 	in6_pcblookup_hash_locked(struct inpcbinfo *pcbinfo,
 	    struct in6_addr *faddr, u_int fport_arg, struct in6_addr *laddr,
-	    u_int lport_arg, int lookupflags, struct ifnet *ifp);
+	    u_int lport_arg, int lookupflags, struct ifnet *ifp, uint8_t);
 struct	inpcb *
 	in6_pcblookup(struct inpcbinfo *, struct in6_addr *,
 			   u_int, struct in6_addr *, u_int, int,



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202012192204.0BJM4kk5092379>