Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 24 Feb 2022 13:04:24 GMT
From:      Marcin Wojtas <mw@FreeBSD.org>
To:        src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-branches@FreeBSD.org
Subject:   git: c749d614b8e4 - stable/12 - ena: merge ena-com v2.5.0 upgrade
Message-ID:  <202202241304.21OD4Ob6044097@gitrepo.freebsd.org>

next in thread | raw e-mail | index | archive | help
The branch stable/12 has been updated by mw:

URL: https://cgit.FreeBSD.org/src/commit/?id=c749d614b8e48a7ec24355046064923d34845cda

commit c749d614b8e48a7ec24355046064923d34845cda
Author:     Marcin Wojtas <mw@FreeBSD.org>
AuthorDate: 2022-01-23 19:21:17 +0000
Commit:     Marcin Wojtas <mw@FreeBSD.org>
CommitDate: 2022-02-24 13:04:04 +0000

    ena: merge ena-com v2.5.0 upgrade
    
    Merge commit '2530eb1fa01bf28fbcfcdda58bd41e055dcb2e4a'
    
    Adjust the driver to the upgraded ena-com part twofold:
    
    First update is related to the driver's NUMA awareness.
    
    Allocate I/O queue memory in NUMA domain local to the CPU bound to the
    given queue, improving data access time. Since this can result in
    performance hit for unaware users, this is done only when RSS
    option is enabled, for other cases the driver relies on kernel to
    allocate memory by itself.
    
    Information about first CPU bound is saved in adapter structure, so
    the binding persists after bringing the interface down and up again.
    
    If there are more buckets than interface queues, the driver will try to
    bind different interfaces to different CPUs using round-robin algorithm
    (but it will not bind queues to CPUs which do not have any RSS buckets
    associated with them). This is done to better utilize hardware
    resources by spreading the load.
    
    Add (read-only) per-queue sysctls in order to provide the following
    information:
    - queueN.domain: NUMA domain associated with the queue
    - queueN.cpu:    CPU affinity of the queue
    
    The second change is for the CSUM_OFFLOAD constant, as ENA platform
    file has removed its definition. To align to that change, it has been
    added to the ena_datapath.h file.
    
    Submitted by: Artur Rojek <ar@semihalf.com>
    Submitted by: Dawid Gorecki <dgr@semihalf.com>
    Obtained from: Semihalf
    MFC after: 2 weeks
    Sponsored by: Amazon, Inc.
    
    (cherry picked from commit eb4c4f4a2e18659b67a6bf1ea5f473c7ed8c854f)
---
 share/man/man4/ena.4           |  5 +++++
 sys/contrib/ena-com/ena_plat.h | 26 ++++++++++++++++++------
 sys/dev/ena/ena.c              | 46 +++++++++++++++++++++++++++++++++++++++---
 sys/dev/ena/ena.h              |  2 ++
 sys/dev/ena/ena_datapath.h     |  2 ++
 sys/dev/ena/ena_sysctl.c       |  8 ++++++++
 6 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/share/man/man4/ena.4 b/share/man/man4/ena.4
index 71bdf4babca6..761d09174391 100644
--- a/share/man/man4/ena.4
+++ b/share/man/man4/ena.4
@@ -71,6 +71,11 @@ is advertised by the device via the Admin Queue), a dedicated MSI-X
 interrupt vector per Tx/Rx queue pair, and CPU cacheline optimized
 data placement.
 .Pp
+When RSS is enabled, each Tx/Rx queue pair is bound to a corresponding
+CPU core and its NUMA domain. The order of those bindings is based on
+the RSS bucket mapping. For builds with RSS support disabled, the
+CPU and NUMA management is left to the kernel.
+.Pp
 The
 .Nm
 driver supports industry standard TCP/IP offload features such
diff --git a/sys/contrib/ena-com/ena_plat.h b/sys/contrib/ena-com/ena_plat.h
index 274f795950c0..9287532b8476 100644
--- a/sys/contrib/ena-com/ena_plat.h
+++ b/sys/contrib/ena-com/ena_plat.h
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/bus.h>
 #include <sys/condvar.h>
+#include <sys/domainset.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/kthread.h>
@@ -170,6 +171,8 @@ static inline long PTR_ERR(const void *ptr)
 #define ENA_COM_TIMER_EXPIRED	ETIMEDOUT
 #define ENA_COM_EIO		EIO
 
+#define ENA_NODE_ANY		(-1)
+
 #define ENA_MSLEEP(x) 		pause_sbt("ena", SBT_1MS * (x), SBT_1MS, 0)
 #define ENA_USLEEP(x) 		pause_sbt("ena", SBT_1US * (x), SBT_1US, 0)
 #define ENA_UDELAY(x) 		DELAY(x)
@@ -277,7 +280,7 @@ typedef struct ifnet ena_netdev;
 void	ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg,
     int error);
 int	ena_dma_alloc(device_t dmadev, bus_size_t size, ena_mem_handle_t *dma,
-    int mapflags, bus_size_t alignment);
+    int mapflags, bus_size_t alignment, int domain);
 
 static inline uint32_t
 ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
@@ -299,16 +302,27 @@ ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
 	} while (0)
 
 #define ENA_MEM_ALLOC(dmadev, size) malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO)
-#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) (virt = NULL)
+
+#define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node)		\
+	do {								\
+		(virt) = malloc_domainset((size), M_DEVBUF,		\
+		    (node) < 0 ? DOMAINSET_RR() : DOMAINSET_PREF(node),	\
+		    M_NOWAIT | M_ZERO);					\
+		(void)(dev_node);					\
+	} while (0)
+
 #define ENA_MEM_FREE(dmadev, ptr, size)					\
 	do { 								\
 		(void)(size);						\
 		free(ptr, M_DEVBUF);					\
 	} while (0)
 #define ENA_MEM_ALLOC_COHERENT_NODE_ALIGNED(dmadev, size, virt, phys,	\
-    handle, node, dev_node, alignment) 					\
+    dma, node, dev_node, alignment) 					\
 	do {								\
-		((virt) = NULL);					\
+		ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment),	\
+		    (node));						\
+		(virt) = (void *)(dma).vaddr;				\
+		(phys) = (dma).paddr;					\
 		(void)(dev_node);					\
 	} while (0)
 
@@ -320,7 +334,8 @@ ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
 #define ENA_MEM_ALLOC_COHERENT_ALIGNED(dmadev, size, virt, phys, dma,	\
     alignment)								\
 	do {								\
-		ena_dma_alloc((dmadev), (size), &(dma), 0, alignment);	\
+		ena_dma_alloc((dmadev), (size), &(dma), 0, (alignment),	\
+		    ENA_NODE_ANY);					\
 		(virt) = (void *)(dma).vaddr;				\
 		(phys) = (dma).paddr;					\
 	} while (0)
@@ -366,7 +381,6 @@ ena_reg_read32(struct ena_bus *bus, bus_size_t offset)
 #define time_after(a,b)	((long)((unsigned long)(b) - (unsigned long)(a)) < 0)
 
 #define VLAN_HLEN 	sizeof(struct ether_vlan_header)
-#define CSUM_OFFLOAD 	(CSUM_IP|CSUM_TCP|CSUM_UDP)
 
 #define prefetch(x)	(void)(x)
 #define prefetchw(x)	(void)(x)
diff --git a/sys/dev/ena/ena.c b/sys/dev/ena/ena.c
index 5b5a7f6593f4..e44c64559128 100644
--- a/sys/dev/ena/ena.c
+++ b/sys/dev/ena/ena.c
@@ -198,7 +198,7 @@ ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 
 int
 ena_dma_alloc(device_t dmadev, bus_size_t size,
-    ena_mem_handle_t *dma, int mapflags, bus_size_t alignment)
+    ena_mem_handle_t *dma, int mapflags, bus_size_t alignment, int domain)
 {
 	struct ena_adapter* adapter = device_get_softc(dmadev);
 	device_t pdev = adapter->pdev;
@@ -229,6 +229,13 @@ ena_dma_alloc(device_t dmadev, bus_size_t size,
 		goto fail_tag;
 	}
 
+	error = bus_dma_tag_set_domain(dma->tag, domain);
+	if (unlikely(error != 0)) {
+		ena_log(pdev, ERR, "bus_dma_tag_set_domain failed: %d\n",
+		    error);
+		goto fail_map_create;
+	}
+
 	error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr,
 	    BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
 	if (unlikely(error != 0)) {
@@ -1445,6 +1452,8 @@ ena_create_io_queues(struct ena_adapter *adapter)
 		ctx.queue_size = adapter->requested_tx_ring_size;
 		ctx.msix_vector = msix_vector;
 		ctx.qid = ena_qid;
+		ctx.numa_node = adapter->que[i].domain;
+
 		rc = ena_com_create_io_queue(ena_dev, &ctx);
 		if (rc != 0) {
 			ena_log(adapter->pdev, ERR,
@@ -1462,6 +1471,11 @@ ena_create_io_queues(struct ena_adapter *adapter)
 			ena_com_destroy_io_queue(ena_dev, ena_qid);
 			goto err_tx;
 		}
+
+		if (ctx.numa_node >= 0) {
+			ena_com_update_numa_node(ring->ena_com_io_cq,
+			    ctx.numa_node);
+		}
 	}
 
 	/* Create RX queues */
@@ -1473,6 +1487,8 @@ ena_create_io_queues(struct ena_adapter *adapter)
 		ctx.queue_size = adapter->requested_rx_ring_size;
 		ctx.msix_vector = msix_vector;
 		ctx.qid = ena_qid;
+		ctx.numa_node = adapter->que[i].domain;
+
 		rc = ena_com_create_io_queue(ena_dev, &ctx);
 		if (unlikely(rc != 0)) {
 			ena_log(adapter->pdev, ERR,
@@ -1491,6 +1507,11 @@ ena_create_io_queues(struct ena_adapter *adapter)
 			ena_com_destroy_io_queue(ena_dev, ena_qid);
 			goto err_rx;
 		}
+
+		if (ctx.numa_node >= 0) {
+			ena_com_update_numa_node(ring->ena_com_io_cq,
+			    ctx.numa_node);
+		}
 	}
 
 	for (i = 0; i < adapter->num_io_queues; i++) {
@@ -1646,12 +1667,22 @@ ena_setup_io_intr(struct ena_adapter *adapter)
 #ifdef RSS
 	int num_buckets = rss_getnumbuckets();
 	static int last_bind = 0;
+	int cur_bind;
+	int idx;
 #endif
 	int irq_idx;
 
 	if (adapter->msix_entries == NULL)
 		return (EINVAL);
 
+#ifdef RSS
+	if (adapter->first_bind < 0) {
+		adapter->first_bind = last_bind;
+		last_bind = (last_bind + adapter->num_io_queues) % num_buckets;
+	}
+	cur_bind = adapter->first_bind;
+#endif
+
 	for (int i = 0; i < adapter->num_io_queues; i++) {
 		irq_idx = ENA_IO_IRQ_IDX(i);
 
@@ -1666,9 +1697,17 @@ ena_setup_io_intr(struct ena_adapter *adapter)
 
 #ifdef RSS
 		adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
-		    rss_getcpu(last_bind);
-		last_bind = (last_bind + 1) % num_buckets;
+		    rss_getcpu(cur_bind);
+		cur_bind = (cur_bind + 1) % num_buckets;
 		CPU_SETOF(adapter->que[i].cpu, &adapter->que[i].cpu_mask);
+
+		for (idx = 0; idx < MAXMEMDOM; ++idx) {
+			if (CPU_ISSET(adapter->que[i].cpu, &cpuset_domain[idx]))
+				break;
+		}
+		adapter->que[i].domain = idx;
+#else
+		adapter->que[i].domain = -1;
 #endif
 	}
 
@@ -3463,6 +3502,7 @@ ena_attach(device_t pdev)
 
 	adapter = device_get_softc(pdev);
 	adapter->pdev = pdev;
+	adapter->first_bind = -1;
 
 	/*
 	 * Set up the timer service - driver is responsible for avoiding
diff --git a/sys/dev/ena/ena.h b/sys/dev/ena/ena.h
index b580bca1159c..476510c8eb67 100644
--- a/sys/dev/ena/ena.h
+++ b/sys/dev/ena/ena.h
@@ -222,6 +222,7 @@ struct ena_que {
 	int cpu;
 	cpuset_t cpu_mask;
 #endif
+	int domain;
 	struct sysctl_oid *oid;
 };
 
@@ -438,6 +439,7 @@ struct ena_adapter {
 	uint32_t buf_ring_size;
 
 	/* RSS*/
+	int first_bind;
 	struct ena_indir *rss_indir;
 
 	uint8_t mac_addr[ETHER_ADDR_LEN];
diff --git a/sys/dev/ena/ena_datapath.h b/sys/dev/ena/ena_datapath.h
index 4886ff1e6391..8da6a2a0edc9 100644
--- a/sys/dev/ena/ena_datapath.h
+++ b/sys/dev/ena/ena_datapath.h
@@ -39,4 +39,6 @@ void	ena_qflush(if_t ifp);
 int	ena_mq_start(if_t ifp, struct mbuf *m);
 void	ena_deferred_mq_start(void *arg, int pending);
 
+#define CSUM_OFFLOAD 	(CSUM_IP|CSUM_TCP|CSUM_UDP)
+
 #endif /* ENA_TXRX_H */
diff --git a/sys/dev/ena/ena_sysctl.c b/sys/dev/ena/ena_sysctl.c
index 4e2b6723dab9..8118e0d0cd93 100644
--- a/sys/dev/ena/ena_sysctl.c
+++ b/sys/dev/ena/ena_sysctl.c
@@ -207,6 +207,14 @@ ena_sysctl_add_stats(struct ena_adapter *adapter)
 
 		adapter->que[i].oid = queue_node;
 
+#ifdef RSS
+		/* Common stats */
+		SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "cpu",
+		    CTLFLAG_RD, &adapter->que[i].cpu, 0, "CPU affinity");
+		SYSCTL_ADD_INT(ctx, queue_list, OID_AUTO, "domain",
+		    CTLFLAG_RD, &adapter->que[i].domain, 0, "NUMA domain");
+#endif
+
 		/* TX specific stats */
 		tx_node = SYSCTL_ADD_NODE(ctx, queue_list, OID_AUTO,
 		    "tx_ring", CTLFLAG_RD, NULL, "TX ring");



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202202241304.21OD4Ob6044097>