Date: Thu, 24 Jun 2021 14:35:52 GMT From: Marcin Wojtas <mw@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: 3fc5d816f883 - main - Merge tag 'vendor/ena-com/2.4.0' Message-ID: <202106241435.15OEZqmR032514@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by mw: URL: https://cgit.FreeBSD.org/src/commit/?id=3fc5d816f8831d6fc2816ac97bd78dc486cd080c commit 3fc5d816f8831d6fc2816ac97bd78dc486cd080c Merge: 0e7d31f63b9d 1f4f67f52424 Author: Marcin Wojtas <mw@FreeBSD.org> AuthorDate: 2021-06-24 14:07:33 +0000 Commit: Marcin Wojtas <mw@FreeBSD.org> CommitDate: 2021-06-24 14:15:18 +0000 Merge tag 'vendor/ena-com/2.4.0' Update the driver in order not to break its compilation and make use of the new ENA logging system Migrate platform code to the new logging system provided by ena_com layer. Make ENA_INFO the new default log level. Remove all explicit use of `device_printf`, all new logs requiring one of the log macros to be used. sys/contrib/ena-com/ena_com.c | 37 +-- sys/contrib/ena-com/ena_com.h | 6 +- sys/contrib/ena-com/ena_defs/ena_admin_defs.h | 23 +- sys/contrib/ena-com/ena_defs/ena_gen_info.h | 6 +- sys/contrib/ena-com/ena_eth_com.c | 4 +- sys/contrib/ena-com/ena_fbsd_log.h | 74 ++++++ sys/contrib/ena-com/ena_plat.h | 37 +-- sys/dev/ena/ena.c | 342 +++++++++++++------------- sys/dev/ena/ena_datapath.c | 83 ++++--- sys/dev/ena/ena_netmap.c | 73 +++--- sys/dev/ena/ena_sysctl.c | 30 +-- 11 files changed, 397 insertions(+), 318 deletions(-) diff --cc sys/contrib/ena-com/ena_fbsd_log.h index 000000000000,311e2f0f280a..311e2f0f280a mode 000000,100644..100644 --- a/sys/contrib/ena-com/ena_fbsd_log.h +++ b/sys/contrib/ena-com/ena_fbsd_log.h diff --cc sys/dev/ena/ena.c index fbe32a7a930e,000000000000..84d58c844332 mode 100644,000000..100644 --- a/sys/dev/ena/ena.c +++ b/sys/dev/ena/ena.c @@@ -1,3980 -1,0 +1,3984 @@@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2015-2021 Amazon.com, Inc. or its affiliates. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_rss.h" + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/bus.h> +#include <sys/endian.h> +#include <sys/kernel.h> +#include <sys/kthread.h> +#include <sys/malloc.h> +#include <sys/mbuf.h> +#include <sys/module.h> +#include <sys/rman.h> +#include <sys/smp.h> +#include <sys/socket.h> +#include <sys/sockio.h> +#include <sys/sysctl.h> +#include <sys/taskqueue.h> +#include <sys/time.h> +#include <sys/eventhandler.h> + +#include <machine/bus.h> +#include <machine/resource.h> +#include <machine/in_cksum.h> + +#include <net/bpf.h> +#include <net/ethernet.h> +#include <net/if.h> +#include <net/if_var.h> +#include <net/if_arp.h> +#include <net/if_dl.h> +#include <net/if_media.h> +#include <net/if_types.h> +#include <net/if_vlan_var.h> +#ifdef RSS +#include <net/rss_config.h> +#endif + +#include <netinet/in_systm.h> +#include <netinet/in.h> +#include <netinet/if_ether.h> +#include <netinet/ip.h> +#include <netinet/ip6.h> +#include <netinet/tcp.h> +#include <netinet/udp.h> + +#include <dev/pci/pcivar.h> +#include <dev/pci/pcireg.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include "ena_datapath.h" +#include "ena.h" +#include "ena_sysctl.h" + +#ifdef DEV_NETMAP +#include "ena_netmap.h" +#endif /* DEV_NETMAP */ + +/********************************************************* + * Function prototypes + *********************************************************/ +static int ena_probe(device_t); +static void ena_intr_msix_mgmnt(void *); +static void ena_free_pci_resources(struct ena_adapter *); +static int ena_change_mtu(if_t, int); +static inline void ena_alloc_counters(counter_u64_t *, int); +static inline void ena_free_counters(counter_u64_t *, int); +static inline void ena_reset_counters(counter_u64_t *, int); +static void ena_init_io_rings_common(struct ena_adapter *, + struct ena_ring *, uint16_t); +static void ena_init_io_rings_basic(struct ena_adapter *); +static void ena_init_io_rings_advanced(struct ena_adapter *); +static void ena_init_io_rings(struct ena_adapter *); +static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int); +static void ena_free_all_io_rings_resources(struct ena_adapter *); +static int ena_setup_tx_dma_tag(struct ena_adapter *); +static int ena_free_tx_dma_tag(struct ena_adapter *); +static int ena_setup_rx_dma_tag(struct ena_adapter *); +static int ena_free_rx_dma_tag(struct ena_adapter *); +static void ena_release_all_tx_dmamap(struct ena_ring *); +static int ena_setup_tx_resources(struct ena_adapter *, int); +static void ena_free_tx_resources(struct ena_adapter *, int); +static int ena_setup_all_tx_resources(struct ena_adapter *); +static void ena_free_all_tx_resources(struct ena_adapter *); +static int ena_setup_rx_resources(struct ena_adapter *, unsigned int); +static void ena_free_rx_resources(struct ena_adapter *, unsigned int); +static int ena_setup_all_rx_resources(struct ena_adapter *); +static void ena_free_all_rx_resources(struct ena_adapter *); +static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *, + struct ena_rx_buffer *); +static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *, + struct ena_rx_buffer *); +static void ena_free_rx_bufs(struct ena_adapter *, unsigned int); +static void ena_refill_all_rx_bufs(struct ena_adapter *); +static void ena_free_all_rx_bufs(struct ena_adapter *); +static void ena_free_tx_bufs(struct ena_adapter *, unsigned int); +static void ena_free_all_tx_bufs(struct ena_adapter *); +static void ena_destroy_all_tx_queues(struct ena_adapter *); +static void ena_destroy_all_rx_queues(struct ena_adapter *); +static void ena_destroy_all_io_queues(struct ena_adapter *); +static int ena_create_io_queues(struct ena_adapter *); +static int ena_handle_msix(void *); +static int ena_enable_msix(struct ena_adapter *); +static void ena_setup_mgmnt_intr(struct ena_adapter *); +static int ena_setup_io_intr(struct ena_adapter *); +static int ena_request_mgmnt_irq(struct ena_adapter *); +static int ena_request_io_irq(struct ena_adapter *); +static void ena_free_mgmnt_irq(struct ena_adapter *); +static void ena_free_io_irq(struct ena_adapter *); +static void ena_free_irqs(struct ena_adapter*); +static void ena_disable_msix(struct ena_adapter *); +static void ena_unmask_all_io_irqs(struct ena_adapter *); +static int ena_rss_configure(struct ena_adapter *); +static int ena_up_complete(struct ena_adapter *); +static uint64_t ena_get_counter(if_t, ift_counter); +static int ena_media_change(if_t); +static void ena_media_status(if_t, struct ifmediareq *); +static void ena_init(void *); +static int ena_ioctl(if_t, u_long, caddr_t); +static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *); +static void ena_update_host_info(struct ena_admin_host_info *, if_t); +static void ena_update_hwassist(struct ena_adapter *); +static int ena_setup_ifnet(device_t, struct ena_adapter *, + struct ena_com_dev_get_features_ctx *); - static int ena_enable_wc(struct resource *); ++static int ena_enable_wc(device_t, struct resource *); +static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *, + struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *); +static uint32_t ena_calc_max_io_queue_num(device_t, struct ena_com_dev *, + struct ena_com_dev_get_features_ctx *); +static int ena_calc_io_queue_size(struct ena_calc_queue_size_ctx *); +static int ena_rss_init_default(struct ena_adapter *); +static void ena_rss_init_default_deferred(void *); +static void ena_config_host_info(struct ena_com_dev *, device_t); +static int ena_attach(device_t); +static int ena_detach(device_t); +static int ena_device_init(struct ena_adapter *, device_t, + struct ena_com_dev_get_features_ctx *, int *); +static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *); +static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *); +static void unimplemented_aenq_handler(void *, + struct ena_admin_aenq_entry *); +static int ena_copy_eni_metrics(struct ena_adapter *); +static void ena_timer_service(void *); + +static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION; + +static ena_vendor_info_t ena_vendor_info_array[] = { + { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0}, + { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF_RSERV0, 0}, + { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0}, + { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF_RSERV0, 0}, + /* Last entry */ + { 0, 0, 0 } +}; + +/* + * Contains pointers to event handlers, e.g. link state chage. + */ +static struct ena_aenq_handlers aenq_handlers; + +void +ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + if (error != 0) + return; + *(bus_addr_t *) arg = segs[0].ds_addr; +} + +int +ena_dma_alloc(device_t dmadev, bus_size_t size, + ena_mem_handle_t *dma, int mapflags, bus_size_t alignment) +{ + struct ena_adapter* adapter = device_get_softc(dmadev); ++ device_t pdev = adapter->pdev; + uint32_t maxsize; + uint64_t dma_space_addr; + int error; + + maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE; + + dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width); + if (unlikely(dma_space_addr == 0)) + dma_space_addr = BUS_SPACE_MAXADDR; + + error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */ + alignment, 0, /* alignment, bounds */ + dma_space_addr, /* lowaddr of exclusion window */ + BUS_SPACE_MAXADDR,/* highaddr of exclusion window */ + NULL, NULL, /* filter, filterarg */ + maxsize, /* maxsize */ + 1, /* nsegments */ + maxsize, /* maxsegsize */ + BUS_DMA_ALLOCNOW, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockarg */ + &dma->tag); + if (unlikely(error != 0)) { - ena_trace(NULL, ENA_ALERT, "bus_dma_tag_create failed: %d\n", error); ++ ena_log(pdev, ERR, "bus_dma_tag_create failed: %d\n", error); + goto fail_tag; + } + + error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr, + BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map); + if (unlikely(error != 0)) { - ena_trace(NULL, ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n", ++ ena_log(pdev, ERR, "bus_dmamem_alloc(%ju) failed: %d\n", + (uintmax_t)size, error); + goto fail_map_create; + } + + dma->paddr = 0; + error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, + size, ena_dmamap_callback, &dma->paddr, mapflags); + if (unlikely((error != 0) || (dma->paddr == 0))) { - ena_trace(NULL, ENA_ALERT, ": bus_dmamap_load failed: %d\n", error); ++ ena_log(pdev, ERR, "bus_dmamap_load failed: %d\n", error); + goto fail_map_load; + } + + bus_dmamap_sync(dma->tag, dma->map, + BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); + + return (0); + +fail_map_load: + bus_dmamem_free(dma->tag, dma->vaddr, dma->map); +fail_map_create: + bus_dma_tag_destroy(dma->tag); +fail_tag: + dma->tag = NULL; + dma->vaddr = NULL; + dma->paddr = 0; + + return (error); +} + +/* + * This function should generate unique key for the whole driver. + * If the key was already genereated in the previous call (for example + * for another adapter), then it should be returned instead. + */ +void +ena_rss_key_fill(void *key, size_t size) +{ + static bool key_generated; + static uint8_t default_key[ENA_HASH_KEY_SIZE]; + + KASSERT(size <= ENA_HASH_KEY_SIZE, ("Requested more bytes than ENA RSS key can hold")); + + if (!key_generated) { + arc4random_buf(default_key, ENA_HASH_KEY_SIZE); + key_generated = true; + } + + memcpy(key, default_key, size); +} + +static void +ena_free_pci_resources(struct ena_adapter *adapter) +{ + device_t pdev = adapter->pdev; + + if (adapter->memory != NULL) { + bus_release_resource(pdev, SYS_RES_MEMORY, + PCIR_BAR(ENA_MEM_BAR), adapter->memory); + } + + if (adapter->registers != NULL) { + bus_release_resource(pdev, SYS_RES_MEMORY, + PCIR_BAR(ENA_REG_BAR), adapter->registers); + } + + if (adapter->msix != NULL) { + bus_release_resource(pdev, SYS_RES_MEMORY, + adapter->msix_rid, adapter->msix); + } +} + +static int +ena_probe(device_t dev) +{ + ena_vendor_info_t *ent; + char adapter_name[60]; + uint16_t pci_vendor_id = 0; + uint16_t pci_device_id = 0; + + pci_vendor_id = pci_get_vendor(dev); + pci_device_id = pci_get_device(dev); + + ent = ena_vendor_info_array; + while (ent->vendor_id != 0) { + if ((pci_vendor_id == ent->vendor_id) && + (pci_device_id == ent->device_id)) { - ena_trace(NULL, ENA_DBG, "vendor=%x device=%x\n", ++ ena_log_raw(DBG, "vendor=%x device=%x\n", + pci_vendor_id, pci_device_id); + + sprintf(adapter_name, DEVICE_DESC); + device_set_desc_copy(dev, adapter_name); + return (BUS_PROBE_DEFAULT); + } + + ent++; + + } + + return (ENXIO); +} + +static int +ena_change_mtu(if_t ifp, int new_mtu) +{ + struct ena_adapter *adapter = if_getsoftc(ifp); ++ device_t pdev = adapter->pdev; + int rc; + + if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) { - device_printf(adapter->pdev, "Invalid MTU setting. " ++ ena_log(pdev, ERR, "Invalid MTU setting. " + "new_mtu: %d max mtu: %d min mtu: %d\n", + new_mtu, adapter->max_mtu, ENA_MIN_MTU); + return (EINVAL); + } + + rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu); + if (likely(rc == 0)) { - ena_trace(NULL, ENA_DBG, "set MTU to %d\n", new_mtu); ++ ena_log(pdev, DBG, "set MTU to %d\n", new_mtu); + if_setmtu(ifp, new_mtu); + } else { - device_printf(adapter->pdev, "Failed to set MTU to %d\n", - new_mtu); ++ ena_log(pdev, ERR, "Failed to set MTU to %d\n", new_mtu); + } + + return (rc); +} + +static inline void +ena_alloc_counters(counter_u64_t *begin, int size) +{ + counter_u64_t *end = (counter_u64_t *)((char *)begin + size); + + for (; begin < end; ++begin) + *begin = counter_u64_alloc(M_WAITOK); +} + +static inline void +ena_free_counters(counter_u64_t *begin, int size) +{ + counter_u64_t *end = (counter_u64_t *)((char *)begin + size); + + for (; begin < end; ++begin) + counter_u64_free(*begin); +} + +static inline void +ena_reset_counters(counter_u64_t *begin, int size) +{ + counter_u64_t *end = (counter_u64_t *)((char *)begin + size); + + for (; begin < end; ++begin) + counter_u64_zero(*begin); +} + +static void +ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring, + uint16_t qid) +{ + + ring->qid = qid; + ring->adapter = adapter; + ring->ena_dev = adapter->ena_dev; + ring->first_interrupt = false; + ring->no_interrupt_event_cnt = 0; +} + +static void +ena_init_io_rings_basic(struct ena_adapter *adapter) +{ + struct ena_com_dev *ena_dev; + struct ena_ring *txr, *rxr; + struct ena_que *que; + int i; + + ena_dev = adapter->ena_dev; + + for (i = 0; i < adapter->num_io_queues; i++) { + txr = &adapter->tx_ring[i]; + rxr = &adapter->rx_ring[i]; + + /* TX/RX common ring state */ + ena_init_io_rings_common(adapter, txr, i); + ena_init_io_rings_common(adapter, rxr, i); + + /* TX specific ring state */ + txr->tx_max_header_size = ena_dev->tx_max_header_size; + txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type; + + que = &adapter->que[i]; + que->adapter = adapter; + que->id = i; + que->tx_ring = txr; + que->rx_ring = rxr; + + txr->que = que; + rxr->que = que; + + rxr->empty_rx_queue = 0; + rxr->rx_mbuf_sz = ena_mbuf_sz; + } +} + +static void +ena_init_io_rings_advanced(struct ena_adapter *adapter) +{ + struct ena_ring *txr, *rxr; + int i; + + for (i = 0; i < adapter->num_io_queues; i++) { + txr = &adapter->tx_ring[i]; + rxr = &adapter->rx_ring[i]; + + /* Allocate a buf ring */ + txr->buf_ring_size = adapter->buf_ring_size; + txr->br = buf_ring_alloc(txr->buf_ring_size, M_DEVBUF, + M_WAITOK, &txr->ring_mtx); + + /* Allocate Tx statistics. */ + ena_alloc_counters((counter_u64_t *)&txr->tx_stats, + sizeof(txr->tx_stats)); + + /* Allocate Rx statistics. */ + ena_alloc_counters((counter_u64_t *)&rxr->rx_stats, + sizeof(rxr->rx_stats)); + + /* Initialize locks */ + snprintf(txr->mtx_name, nitems(txr->mtx_name), "%s:tx(%d)", + device_get_nameunit(adapter->pdev), i); + snprintf(rxr->mtx_name, nitems(rxr->mtx_name), "%s:rx(%d)", + device_get_nameunit(adapter->pdev), i); + + mtx_init(&txr->ring_mtx, txr->mtx_name, NULL, MTX_DEF); + } +} + +static void +ena_init_io_rings(struct ena_adapter *adapter) +{ + /* + * IO rings initialization can be divided into the 2 steps: + * 1. Initialize variables and fields with initial values and copy + * them from adapter/ena_dev (basic) + * 2. Allocate mutex, counters and buf_ring (advanced) + */ + ena_init_io_rings_basic(adapter); + ena_init_io_rings_advanced(adapter); +} + +static void +ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid) +{ + struct ena_ring *txr = &adapter->tx_ring[qid]; + struct ena_ring *rxr = &adapter->rx_ring[qid]; + + ena_free_counters((counter_u64_t *)&txr->tx_stats, + sizeof(txr->tx_stats)); + ena_free_counters((counter_u64_t *)&rxr->rx_stats, + sizeof(rxr->rx_stats)); + + ENA_RING_MTX_LOCK(txr); + drbr_free(txr->br, M_DEVBUF); + ENA_RING_MTX_UNLOCK(txr); + + mtx_destroy(&txr->ring_mtx); +} + +static void +ena_free_all_io_rings_resources(struct ena_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_io_queues; i++) + ena_free_io_ring_resources(adapter, i); + +} + +static int +ena_setup_tx_dma_tag(struct ena_adapter *adapter) +{ + int ret; + + /* Create DMA tag for Tx buffers */ + ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), + 1, 0, /* alignment, bounds */ + ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ + BUS_SPACE_MAXADDR, /* highaddr of excl window */ + NULL, NULL, /* filter, filterarg */ + ENA_TSO_MAXSIZE, /* maxsize */ + adapter->max_tx_sgl_size - 1, /* nsegments */ + ENA_TSO_MAXSIZE, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockfuncarg */ + &adapter->tx_buf_tag); + + return (ret); +} + +static int +ena_free_tx_dma_tag(struct ena_adapter *adapter) +{ + int ret; + + ret = bus_dma_tag_destroy(adapter->tx_buf_tag); + + if (likely(ret == 0)) + adapter->tx_buf_tag = NULL; + + return (ret); +} + +static int +ena_setup_rx_dma_tag(struct ena_adapter *adapter) +{ + int ret; + + /* Create DMA tag for Rx buffers*/ + ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */ + 1, 0, /* alignment, bounds */ + ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */ + BUS_SPACE_MAXADDR, /* highaddr of excl window */ + NULL, NULL, /* filter, filterarg */ + ena_mbuf_sz, /* maxsize */ + adapter->max_rx_sgl_size, /* nsegments */ + ena_mbuf_sz, /* maxsegsize */ + 0, /* flags */ + NULL, /* lockfunc */ + NULL, /* lockarg */ + &adapter->rx_buf_tag); + + return (ret); +} + +static int +ena_free_rx_dma_tag(struct ena_adapter *adapter) +{ + int ret; + + ret = bus_dma_tag_destroy(adapter->rx_buf_tag); + + if (likely(ret == 0)) + adapter->rx_buf_tag = NULL; + + return (ret); +} + +static void +ena_release_all_tx_dmamap(struct ena_ring *tx_ring) +{ + struct ena_adapter *adapter = tx_ring->adapter; + struct ena_tx_buffer *tx_info; + bus_dma_tag_t tx_tag = adapter->tx_buf_tag;; + int i; +#ifdef DEV_NETMAP + struct ena_netmap_tx_info *nm_info; + int j; +#endif /* DEV_NETMAP */ + + for (i = 0; i < tx_ring->ring_size; ++i) { + tx_info = &tx_ring->tx_buffer_info[i]; +#ifdef DEV_NETMAP + if (adapter->ifp->if_capenable & IFCAP_NETMAP) { + nm_info = &tx_info->nm_info; + for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) { + if (nm_info->map_seg[j] != NULL) { + bus_dmamap_destroy(tx_tag, + nm_info->map_seg[j]); + nm_info->map_seg[j] = NULL; + } + } + } +#endif /* DEV_NETMAP */ + if (tx_info->dmamap != NULL) { + bus_dmamap_destroy(tx_tag, tx_info->dmamap); + tx_info->dmamap = NULL; + } + } +} + +/** + * ena_setup_tx_resources - allocate Tx resources (Descriptors) + * @adapter: network interface device structure + * @qid: queue index + * + * Returns 0 on success, otherwise on failure. + **/ +static int +ena_setup_tx_resources(struct ena_adapter *adapter, int qid) +{ ++ device_t pdev = adapter->pdev; + struct ena_que *que = &adapter->que[qid]; + struct ena_ring *tx_ring = que->tx_ring; + int size, i, err; +#ifdef DEV_NETMAP + bus_dmamap_t *map; + int j; + + ena_netmap_reset_tx_ring(adapter, qid); +#endif /* DEV_NETMAP */ + + size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size; + + tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); + if (unlikely(tx_ring->tx_buffer_info == NULL)) + return (ENOMEM); + + size = sizeof(uint16_t) * tx_ring->ring_size; + tx_ring->free_tx_ids = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); + if (unlikely(tx_ring->free_tx_ids == NULL)) + goto err_buf_info_free; + + size = tx_ring->tx_max_header_size; + tx_ring->push_buf_intermediate_buf = malloc(size, M_DEVBUF, + M_NOWAIT | M_ZERO); + if (unlikely(tx_ring->push_buf_intermediate_buf == NULL)) + goto err_tx_ids_free; + + /* Req id stack for TX OOO completions */ + for (i = 0; i < tx_ring->ring_size; i++) + tx_ring->free_tx_ids[i] = i; + + /* Reset TX statistics. */ + ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats, + sizeof(tx_ring->tx_stats)); + + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; + tx_ring->acum_pkts = 0; + + /* Make sure that drbr is empty */ + ENA_RING_MTX_LOCK(tx_ring); + drbr_flush(adapter->ifp, tx_ring->br); + ENA_RING_MTX_UNLOCK(tx_ring); + + /* ... and create the buffer DMA maps */ + for (i = 0; i < tx_ring->ring_size; i++) { + err = bus_dmamap_create(adapter->tx_buf_tag, 0, + &tx_ring->tx_buffer_info[i].dmamap); + if (unlikely(err != 0)) { - ena_trace(NULL, ENA_ALERT, ++ ena_log(pdev, ERR, + "Unable to create Tx DMA map for buffer %d\n", + i); + goto err_map_release; + } + +#ifdef DEV_NETMAP + if (adapter->ifp->if_capenable & IFCAP_NETMAP) { + map = tx_ring->tx_buffer_info[i].nm_info.map_seg; + for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { + err = bus_dmamap_create(adapter->tx_buf_tag, 0, + &map[j]); + if (unlikely(err != 0)) { - ena_trace(NULL, ENA_ALERT, "Unable to create " ++ ena_log(pdev, ERR, ++ "Unable to create " + "Tx DMA for buffer %d %d\n", i, j); + goto err_map_release; + } + } + } +#endif /* DEV_NETMAP */ + } + + /* Allocate taskqueues */ + TASK_INIT(&tx_ring->enqueue_task, 0, ena_deferred_mq_start, tx_ring); + tx_ring->enqueue_tq = taskqueue_create_fast("ena_tx_enque", M_NOWAIT, + taskqueue_thread_enqueue, &tx_ring->enqueue_tq); + if (unlikely(tx_ring->enqueue_tq == NULL)) { - ena_trace(NULL, ENA_ALERT, ++ ena_log(pdev, ERR, + "Unable to create taskqueue for enqueue task\n"); + i = tx_ring->ring_size; + goto err_map_release; + } + + tx_ring->running = true; + + taskqueue_start_threads(&tx_ring->enqueue_tq, 1, PI_NET, + "%s txeq %d", device_get_nameunit(adapter->pdev), que->cpu); + + return (0); + +err_map_release: + ena_release_all_tx_dmamap(tx_ring); +err_tx_ids_free: + free(tx_ring->free_tx_ids, M_DEVBUF); + tx_ring->free_tx_ids = NULL; +err_buf_info_free: + free(tx_ring->tx_buffer_info, M_DEVBUF); + tx_ring->tx_buffer_info = NULL; + + return (ENOMEM); +} + +/** + * ena_free_tx_resources - Free Tx Resources per Queue + * @adapter: network interface device structure + * @qid: queue index + * + * Free all transmit software resources + **/ +static void +ena_free_tx_resources(struct ena_adapter *adapter, int qid) +{ + struct ena_ring *tx_ring = &adapter->tx_ring[qid]; +#ifdef DEV_NETMAP + struct ena_netmap_tx_info *nm_info; + int j; +#endif /* DEV_NETMAP */ + + while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task, + NULL)) + taskqueue_drain(tx_ring->enqueue_tq, &tx_ring->enqueue_task); + + taskqueue_free(tx_ring->enqueue_tq); + + ENA_RING_MTX_LOCK(tx_ring); + /* Flush buffer ring, */ + drbr_flush(adapter->ifp, tx_ring->br); + + /* Free buffer DMA maps, */ + for (int i = 0; i < tx_ring->ring_size; i++) { + bus_dmamap_sync(adapter->tx_buf_tag, + tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(adapter->tx_buf_tag, + tx_ring->tx_buffer_info[i].dmamap); + bus_dmamap_destroy(adapter->tx_buf_tag, + tx_ring->tx_buffer_info[i].dmamap); + +#ifdef DEV_NETMAP + if (adapter->ifp->if_capenable & IFCAP_NETMAP) { + nm_info = &tx_ring->tx_buffer_info[i].nm_info; + for (j = 0; j < ENA_PKT_MAX_BUFS; j++) { + if (nm_info->socket_buf_idx[j] != 0) { + bus_dmamap_sync(adapter->tx_buf_tag, + nm_info->map_seg[j], + BUS_DMASYNC_POSTWRITE); + ena_netmap_unload(adapter, + nm_info->map_seg[j]); + } + bus_dmamap_destroy(adapter->tx_buf_tag, + nm_info->map_seg[j]); + nm_info->socket_buf_idx[j] = 0; + } + } +#endif /* DEV_NETMAP */ + + m_freem(tx_ring->tx_buffer_info[i].mbuf); + tx_ring->tx_buffer_info[i].mbuf = NULL; + } + ENA_RING_MTX_UNLOCK(tx_ring); + + /* And free allocated memory. */ + free(tx_ring->tx_buffer_info, M_DEVBUF); + tx_ring->tx_buffer_info = NULL; + + free(tx_ring->free_tx_ids, M_DEVBUF); + tx_ring->free_tx_ids = NULL; + + free(tx_ring->push_buf_intermediate_buf, M_DEVBUF); + tx_ring->push_buf_intermediate_buf = NULL; +} + +/** + * ena_setup_all_tx_resources - allocate all queues Tx resources + * @adapter: network interface device structure + * + * Returns 0 on success, otherwise on failure. + **/ +static int +ena_setup_all_tx_resources(struct ena_adapter *adapter) +{ + int i, rc; + + for (i = 0; i < adapter->num_io_queues; i++) { + rc = ena_setup_tx_resources(adapter, i); + if (rc != 0) { - device_printf(adapter->pdev, ++ ena_log(adapter->pdev, ERR, + "Allocation for Tx Queue %u failed\n", i); + goto err_setup_tx; + } + } + + return (0); + +err_setup_tx: + /* Rewind the index freeing the rings as we go */ + while (i--) + ena_free_tx_resources(adapter, i); + return (rc); +} + +/** + * ena_free_all_tx_resources - Free Tx Resources for All Queues + * @adapter: network interface device structure + * + * Free all transmit software resources + **/ +static void +ena_free_all_tx_resources(struct ena_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_io_queues; i++) + ena_free_tx_resources(adapter, i); +} + +/** + * ena_setup_rx_resources - allocate Rx resources (Descriptors) + * @adapter: network interface device structure + * @qid: queue index + * + * Returns 0 on success, otherwise on failure. + **/ +static int +ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid) +{ ++ device_t pdev = adapter->pdev; + struct ena_que *que = &adapter->que[qid]; + struct ena_ring *rx_ring = que->rx_ring; + int size, err, i; + + size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size; + +#ifdef DEV_NETMAP + ena_netmap_reset_rx_ring(adapter, qid); + rx_ring->initialized = false; +#endif /* DEV_NETMAP */ + + /* + * Alloc extra element so in rx path + * we can always prefetch rx_info + 1 + */ + size += sizeof(struct ena_rx_buffer); + + rx_ring->rx_buffer_info = malloc(size, M_DEVBUF, M_WAITOK | M_ZERO); + + size = sizeof(uint16_t) * rx_ring->ring_size; + rx_ring->free_rx_ids = malloc(size, M_DEVBUF, M_WAITOK); + + for (i = 0; i < rx_ring->ring_size; i++) + rx_ring->free_rx_ids[i] = i; + + /* Reset RX statistics. */ + ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats, + sizeof(rx_ring->rx_stats)); + + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; + + /* ... and create the buffer DMA maps */ + for (i = 0; i < rx_ring->ring_size; i++) { + err = bus_dmamap_create(adapter->rx_buf_tag, 0, + &(rx_ring->rx_buffer_info[i].map)); + if (err != 0) { - ena_trace(NULL, ENA_ALERT, ++ ena_log(pdev, ERR, + "Unable to create Rx DMA map for buffer %d\n", i); + goto err_buf_info_unmap; + } + } + + /* Create LRO for the ring */ + if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) { + int err = tcp_lro_init(&rx_ring->lro); + if (err != 0) { - device_printf(adapter->pdev, - "LRO[%d] Initialization failed!\n", qid); ++ ena_log(pdev, ERR, "LRO[%d] Initialization failed!\n", ++ qid); + } else { - ena_trace(NULL, ENA_INFO, - "RX Soft LRO[%d] Initialized\n", qid); ++ ena_log(pdev, DBG, "RX Soft LRO[%d] Initialized\n", ++ qid); + rx_ring->lro.ifp = adapter->ifp; + } + } + + return (0); + +err_buf_info_unmap: + while (i--) { + bus_dmamap_destroy(adapter->rx_buf_tag, + rx_ring->rx_buffer_info[i].map); + } + + free(rx_ring->free_rx_ids, M_DEVBUF); + rx_ring->free_rx_ids = NULL; + free(rx_ring->rx_buffer_info, M_DEVBUF); + rx_ring->rx_buffer_info = NULL; *** 6157 LINES SKIPPED ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202106241435.15OEZqmR032514>