From owner-svn-src-all@freebsd.org Sat May 21 15:15:13 2016 Return-Path: Delivered-To: svn-src-all@mailman.ysv.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:1900:2254:206a::19:1]) by mailman.ysv.freebsd.org (Postfix) with ESMTP id 7BA00B44A94; Sat, 21 May 2016 15:15:13 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id 04BB91777; Sat, 21 May 2016 15:15:12 +0000 (UTC) (envelope-from mav@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id u4LFFCix082907; Sat, 21 May 2016 15:15:12 GMT (envelope-from mav@FreeBSD.org) Received: (from mav@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id u4LFFCMK082904; Sat, 21 May 2016 15:15:12 GMT (envelope-from mav@FreeBSD.org) Message-Id: <201605211515.u4LFFCMK082904@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mav set sender to mav@FreeBSD.org using -f From: Alexander Motin Date: Sat, 21 May 2016 15:15:12 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r300373 - in stable/10/sys: conf dev/ntb/if_ntb dev/ntb/ntb_hw X-SVN-Group: stable-10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.22 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 21 May 2016 15:15:13 -0000 Author: mav Date: Sat May 21 15:15:11 2016 New Revision: 300373 URL: https://svnweb.freebsd.org/changeset/base/300373 Log: MFC 103 ntb(4) patches by cem@ up to r295487. Modified: stable/10/sys/conf/files.i386 stable/10/sys/dev/ntb/if_ntb/if_ntb.c stable/10/sys/dev/ntb/ntb_hw/ntb_hw.c stable/10/sys/dev/ntb/ntb_hw/ntb_hw.h stable/10/sys/dev/ntb/ntb_hw/ntb_regs.h Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/conf/files.i386 ============================================================================== --- stable/10/sys/conf/files.i386 Sat May 21 14:51:49 2016 (r300372) +++ stable/10/sys/conf/files.i386 Sat May 21 15:15:11 2016 (r300373) @@ -274,6 +274,8 @@ dev/lindev/lindev.c optional lindev dev/mse/mse.c optional mse dev/mse/mse_isa.c optional mse isa dev/nfe/if_nfe.c optional nfe pci +dev/ntb/if_ntb/if_ntb.c optional if_ntb +dev/ntb/ntb_hw/ntb_hw.c optional if_ntb | ntb_hw dev/nvd/nvd.c optional nvd nvme dev/nve/if_nve.c optional nve pci dev/nvme/nvme.c optional nvme Modified: stable/10/sys/dev/ntb/if_ntb/if_ntb.c ============================================================================== --- stable/10/sys/dev/ntb/if_ntb/if_ntb.c Sat May 21 14:51:49 2016 (r300372) +++ stable/10/sys/dev/ntb/if_ntb/if_ntb.c Sat May 21 15:15:11 2016 (r300373) @@ -1,5 +1,6 @@ /*- * Copyright (C) 2013 Intel Corporation + * Copyright (C) 2015 EMC Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,8 +31,10 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include +#include #include #include #include @@ -39,19 +42,26 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include + #include #include #include #include #include #include + #include #include + #include #include #include +#include +#include + #include "../ntb_hw/ntb_hw.h" /* @@ -65,72 +75,113 @@ __FBSDID("$FreeBSD$"); * be picked up and redistributed in Linux with a dual GPL/BSD license. */ -/* TODO: These functions should really be part of the kernel */ -#define test_bit(pos, bitmap_addr) (*(bitmap_addr) & 1UL << (pos)) -#define set_bit(pos, bitmap_addr) *(bitmap_addr) |= 1UL << (pos) -#define clear_bit(pos, bitmap_addr) *(bitmap_addr) &= ~(1UL << (pos)) +#define QP_SETSIZE 64 +BITSET_DEFINE(_qpset, QP_SETSIZE); +#define test_bit(pos, addr) BIT_ISSET(QP_SETSIZE, (pos), (addr)) +#define set_bit(pos, addr) BIT_SET(QP_SETSIZE, (pos), (addr)) +#define clear_bit(pos, addr) BIT_CLR(QP_SETSIZE, (pos), (addr)) +#define ffs_bit(addr) BIT_FFS(QP_SETSIZE, (addr)) #define KTR_NTB KTR_SPARE3 -#define NTB_TRANSPORT_VERSION 3 +#define NTB_TRANSPORT_VERSION 4 #define NTB_RX_MAX_PKTS 64 #define NTB_RXQ_SIZE 300 -static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN; -static unsigned int max_num_clients = 1; +enum ntb_link_event { + NTB_LINK_DOWN = 0, + NTB_LINK_UP, +}; + +static SYSCTL_NODE(_hw, OID_AUTO, if_ntb, CTLFLAG_RW, 0, "if_ntb"); + +static unsigned g_if_ntb_debug_level; +SYSCTL_UINT(_hw_if_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, + &g_if_ntb_debug_level, 0, "if_ntb log level -- higher is more verbose"); +#define ntb_printf(lvl, ...) do { \ + if ((lvl) <= g_if_ntb_debug_level) { \ + if_printf(nt->ifp, __VA_ARGS__); \ + } \ +} while (0) + +static unsigned transport_mtu = IP_MAXPACKET + ETHER_HDR_LEN + ETHER_CRC_LEN; + +static uint64_t max_mw_size; +SYSCTL_UQUAD(_hw_if_ntb, OID_AUTO, max_mw_size, CTLFLAG_RDTUN, &max_mw_size, 0, + "If enabled (non-zero), limit the size of large memory windows. " + "Both sides of the NTB MUST set the same value here."); + +static unsigned max_num_clients; +SYSCTL_UINT(_hw_if_ntb, OID_AUTO, max_num_clients, CTLFLAG_RDTUN, + &max_num_clients, 0, "Maximum number of NTB transport clients. " + "0 (default) - use all available NTB memory windows; " + "positive integer N - Limit to N memory windows."); + +static unsigned enable_xeon_watchdog; +SYSCTL_UINT(_hw_if_ntb, OID_AUTO, enable_xeon_watchdog, CTLFLAG_RDTUN, + &enable_xeon_watchdog, 0, "If non-zero, write a register every second to " + "keep a watchdog from tearing down the NTB link"); STAILQ_HEAD(ntb_queue_list, ntb_queue_entry); +typedef uint32_t ntb_q_idx_t; + struct ntb_queue_entry { /* ntb_queue list reference */ STAILQ_ENTRY(ntb_queue_entry) entry; - /* info on data to be transfered */ + /* info on data to be transferred */ void *cb_data; void *buf; - uint64_t len; - uint64_t flags; + uint32_t len; + uint32_t flags; + + struct ntb_transport_qp *qp; + struct ntb_payload_header *x_hdr; + ntb_q_idx_t index; }; struct ntb_rx_info { - unsigned int entry; + ntb_q_idx_t entry; }; struct ntb_transport_qp { - struct ntb_netdev *transport; + struct ntb_transport_ctx *transport; struct ntb_softc *ntb; void *cb_data; bool client_ready; - bool qp_link; + volatile bool link_is_up; uint8_t qp_num; /* Only 64 QPs are allowed. 0-63 */ struct ntb_rx_info *rx_info; struct ntb_rx_info *remote_rx_info; - void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, + void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, void *data, int len); struct ntb_queue_list tx_free_q; struct mtx ntb_tx_free_q_lock; - void *tx_mw; - uint64_t tx_index; - uint64_t tx_max_entry; + caddr_t tx_mw; + bus_addr_t tx_mw_phys; + ntb_q_idx_t tx_index; + ntb_q_idx_t tx_max_entry; uint64_t tx_max_frame; - void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, + void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, void *data, int len); + struct ntb_queue_list rx_post_q; struct ntb_queue_list rx_pend_q; - struct ntb_queue_list rx_free_q; - struct mtx ntb_rx_pend_q_lock; - struct mtx ntb_rx_free_q_lock; + /* ntb_rx_q_lock: synchronize access to rx_XXXX_q */ + struct mtx ntb_rx_q_lock; struct task rx_completion_task; - void *rx_buff; - uint64_t rx_index; - uint64_t rx_max_entry; + struct task rxc_db_work; + caddr_t rx_buff; + ntb_q_idx_t rx_index; + ntb_q_idx_t rx_max_entry; uint64_t rx_max_frame; - void (*event_handler) (void *data, int status); + void (*event_handler)(void *data, enum ntb_link_event status); struct callout link_work; struct callout queue_full; struct callout rx_full; @@ -147,40 +198,55 @@ struct ntb_transport_qp { uint64_t tx_bytes; uint64_t tx_pkts; uint64_t tx_ring_full; + uint64_t tx_err_no_buf; }; struct ntb_queue_handlers { - void (*rx_handler) (struct ntb_transport_qp *qp, void *qp_data, + void (*rx_handler)(struct ntb_transport_qp *qp, void *qp_data, void *data, int len); - void (*tx_handler) (struct ntb_transport_qp *qp, void *qp_data, + void (*tx_handler)(struct ntb_transport_qp *qp, void *qp_data, void *data, int len); - void (*event_handler) (void *data, int status); + void (*event_handler)(void *data, enum ntb_link_event status); }; - struct ntb_transport_mw { - size_t size; - void *virt_addr; - vm_paddr_t dma_addr; + vm_paddr_t phys_addr; + size_t phys_size; + size_t xlat_align; + size_t xlat_align_size; + bus_addr_t addr_limit; + /* Tx buff is off vbase / phys_addr */ + caddr_t vbase; + size_t xlat_size; + size_t buff_size; + /* Rx buff is off virt_addr / dma_addr */ + caddr_t virt_addr; + bus_addr_t dma_addr; }; -struct ntb_netdev { +struct ntb_transport_ctx { struct ntb_softc *ntb; struct ifnet *ifp; - struct ntb_transport_mw mw[NTB_NUM_MW]; - struct ntb_transport_qp *qps; - uint64_t max_qps; - uint64_t qp_bitmap; - bool transport_link; + struct ntb_transport_mw mw_vec[NTB_MAX_NUM_MW]; + struct ntb_transport_qp *qp_vec; + struct _qpset qp_bitmap; + struct _qpset qp_bitmap_free; + unsigned mw_count; + unsigned qp_count; + volatile bool link_is_up; struct callout link_work; - struct ntb_transport_qp *qp; + struct callout link_watchdog; + struct task link_cleanup; uint64_t bufsize; u_char eaddr[ETHER_ADDR_LEN]; struct mtx tx_lock; struct mtx rx_lock; + + /* The hardcoded single queuepair in ntb_setup_interface() */ + struct ntb_transport_qp *qp; }; -static struct ntb_netdev net_softc; +static struct ntb_transport_ctx net_softc; enum { IF_NTB_DESC_DONE_FLAG = 1 << 0, @@ -188,21 +254,42 @@ enum { }; struct ntb_payload_header { - uint64_t ver; - uint64_t len; - uint64_t flags; + ntb_q_idx_t ver; + uint32_t len; + uint32_t flags; }; enum { + /* + * The order of this enum is part of the if_ntb remote protocol. Do + * not reorder without bumping protocol version (and it's probably best + * to keep the protocol in lock-step with the Linux NTB driver. + */ IF_NTB_VERSION = 0, - IF_NTB_MW0_SZ, - IF_NTB_MW1_SZ, - IF_NTB_NUM_QPS, IF_NTB_QP_LINKS, + IF_NTB_NUM_QPS, + IF_NTB_NUM_MWS, + /* + * N.B.: transport_link_work assumes MW1 enums = MW0 + 2. + */ + IF_NTB_MW0_SZ_HIGH, + IF_NTB_MW0_SZ_LOW, + IF_NTB_MW1_SZ_HIGH, + IF_NTB_MW1_SZ_LOW, IF_NTB_MAX_SPAD, + + /* + * Some NTB-using hardware have a watchdog to work around NTB hangs; if + * a register or doorbell isn't written every few seconds, the link is + * torn down. Write an otherwise unused register every few seconds to + * work around this watchdog. + */ + IF_NTB_WATCHDOG_SPAD = 15 }; +CTASSERT(IF_NTB_WATCHDOG_SPAD < XEON_SPAD_COUNT && + IF_NTB_WATCHDOG_SPAD < ATOM_SPAD_COUNT); -#define QP_TO_MW(qp) ((qp) % NTB_NUM_MW) +#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count) #define NTB_QP_DEF_NUM_ENTRIES 100 #define NTB_LINK_DOWN_TIMEOUT 10 @@ -216,37 +303,41 @@ static void ntb_net_tx_handler(struct nt void *data, int len); static void ntb_net_rx_handler(struct ntb_transport_qp *qp, void *qp_data, void *data, int len); -static void ntb_net_event_handler(void *data, int status); -static int ntb_transport_init(struct ntb_softc *ntb); -static void ntb_transport_free(void *transport); -static void ntb_transport_init_queue(struct ntb_netdev *nt, +static void ntb_net_event_handler(void *data, enum ntb_link_event status); +static int ntb_transport_probe(struct ntb_softc *ntb); +static void ntb_transport_free(struct ntb_transport_ctx *); +static void ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num); static void ntb_transport_free_queue(struct ntb_transport_qp *qp); -static struct ntb_transport_qp * ntb_transport_create_queue(void *data, +static struct ntb_transport_qp *ntb_transport_create_queue(void *data, struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers); static void ntb_transport_link_up(struct ntb_transport_qp *qp); static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb, void *data, unsigned int len); static int ntb_process_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry); -static void ntb_tx_copy_task(struct ntb_transport_qp *qp, +static void ntb_memcpy_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, void *offset); static void ntb_qp_full(void *arg); -static void ntb_transport_rxc_db(void *data, int db_num); -static void ntb_rx_pendq_full(void *arg); -static void ntb_transport_rx(struct ntb_transport_qp *qp); +static void ntb_transport_rxc_db(void *arg, int pending); static int ntb_process_rxc(struct ntb_transport_qp *qp); -static void ntb_rx_copy_task(struct ntb_transport_qp *qp, +static void ntb_memcpy_rx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, void *offset); -static void ntb_rx_completion_task(void *arg, int pending); -static void ntb_transport_event_callback(void *data, enum ntb_hw_event event); +static inline void ntb_rx_copy_callback(struct ntb_transport_qp *qp, + void *data); +static void ntb_complete_rxc(void *arg, int pending); +static void ntb_transport_doorbell_callback(void *data, uint32_t vector); +static void ntb_transport_event_callback(void *data); static void ntb_transport_link_work(void *arg); -static int ntb_set_mw(struct ntb_netdev *nt, int num_mw, unsigned int size); -static void ntb_transport_setup_qp_mw(struct ntb_netdev *nt, +static int ntb_set_mw(struct ntb_transport_ctx *, int num_mw, size_t size); +static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw); +static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num); static void ntb_qp_link_work(void *arg); -static void ntb_transport_link_cleanup(struct ntb_netdev *nt); +static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt); +static void ntb_transport_link_cleanup_work(void *, int); static void ntb_qp_link_down(struct ntb_transport_qp *qp); +static void ntb_qp_link_down_reset(struct ntb_transport_qp *qp); static void ntb_qp_link_cleanup(struct ntb_transport_qp *qp); static void ntb_transport_link_down(struct ntb_transport_qp *qp); static void ntb_send_link_down(struct ntb_transport_qp *qp); @@ -254,11 +345,27 @@ static void ntb_list_add(struct mtx *loc struct ntb_queue_list *list); static struct ntb_queue_entry *ntb_list_rm(struct mtx *lock, struct ntb_queue_list *list); +static struct ntb_queue_entry *ntb_list_mv(struct mtx *lock, + struct ntb_queue_list *from, struct ntb_queue_list *to); static void create_random_local_eui48(u_char *eaddr); static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp); +static void xeon_link_watchdog_hb(void *); + +static const struct ntb_ctx_ops ntb_transport_ops = { + .link_event = ntb_transport_event_callback, + .db_event = ntb_transport_doorbell_callback, +}; MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver"); +static inline void +iowrite32(uint32_t val, void *addr) +{ + + bus_space_write_4(X86_BUS_SPACE_MEM, 0/* HACK */, (uintptr_t)addr, + val); +} + /* Module setup and teardown */ static int ntb_handle_module_events(struct module *m, int what, void *arg) @@ -289,11 +396,12 @@ DECLARE_MODULE(if_ntb, if_ntb_mod, SI_SU MODULE_DEPEND(if_ntb, ntb_hw, 1, 1, 1); static int -ntb_setup_interface() +ntb_setup_interface(void) { struct ifnet *ifp; struct ntb_queue_handlers handlers = { ntb_net_rx_handler, ntb_net_tx_handler, ntb_net_event_handler }; + int rc; net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0); if (net_softc.ntb == NULL) { @@ -301,17 +409,23 @@ ntb_setup_interface() return (ENXIO); } - ntb_transport_init(net_softc.ntb); - ifp = net_softc.ifp = if_alloc(IFT_ETHER); if (ifp == NULL) { - printf("ntb: cannot allocate ifnet structure\n"); + ntb_transport_free(&net_softc); + printf("ntb: Cannot allocate ifnet structure\n"); return (ENOMEM); } + if_initname(ifp, "ntb", 0); + + rc = ntb_transport_probe(net_softc.ntb); + if (rc != 0) { + printf("ntb: Cannot init transport: %d\n", rc); + if_free(net_softc.ifp); + return (rc); + } net_softc.qp = ntb_transport_create_queue(ifp, net_softc.ntb, &handlers); - if_initname(ifp, "ntb", 0); ifp->if_init = ntb_net_init; ifp->if_softc = &net_softc; ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX; @@ -324,6 +438,8 @@ ntb_setup_interface() ether_ifattach(ifp, net_softc.eaddr); ifp->if_capabilities = IFCAP_HWCSUM | IFCAP_JUMBO_MTU; ifp->if_capenable = ifp->if_capabilities; + ifp->if_mtu = ntb_transport_max_size(net_softc.qp) - ETHER_HDR_LEN - + ETHER_CRC_LEN; ntb_transport_link_up(net_softc.qp); net_softc.bufsize = ntb_transport_max_size(net_softc.qp) + @@ -332,20 +448,20 @@ ntb_setup_interface() } static int -ntb_teardown_interface() +ntb_teardown_interface(void) { - if (net_softc.qp != NULL) + if (net_softc.qp != NULL) { ntb_transport_link_down(net_softc.qp); + ntb_transport_free_queue(net_softc.qp); + ntb_transport_free(&net_softc); + } + if (net_softc.ifp != NULL) { ether_ifdetach(net_softc.ifp); if_free(net_softc.ifp); - } - - if (net_softc.qp != NULL) { - ntb_transport_free_queue(net_softc.qp); - ntb_transport_free(&net_softc); + net_softc.ifp = NULL; } return (0); @@ -356,7 +472,7 @@ ntb_teardown_interface() static void ntb_net_init(void *arg) { - struct ntb_netdev *ntb_softc = arg; + struct ntb_transport_ctx *ntb_softc = arg; struct ifnet *ifp = ntb_softc->ifp; ifp->if_drv_flags |= IFF_DRV_RUNNING; @@ -368,7 +484,7 @@ ntb_net_init(void *arg) static int ntb_ioctl(struct ifnet *ifp, u_long command, caddr_t data) { - struct ntb_netdev *nt = ifp->if_softc; + struct ntb_transport_ctx *nt = ifp->if_softc; struct ifreq *ifr = (struct ifreq *)data; int error = 0; @@ -397,7 +513,7 @@ static void ntb_start(struct ifnet *ifp) { struct mbuf *m_head; - struct ntb_netdev *nt = ifp->if_softc; + struct ntb_transport_ctx *nt = ifp->if_softc; int rc; mtx_lock(&nt->tx_lock); @@ -447,125 +563,199 @@ ntb_net_rx_handler(struct ntb_transport_ } static void -ntb_net_event_handler(void *data, int status) +ntb_net_event_handler(void *data, enum ntb_link_event status) { + struct ifnet *ifp; + + ifp = data; + (void)ifp; + + /* XXX The Linux driver munges with the carrier status here. */ + switch (status) { + case NTB_LINK_DOWN: + break; + case NTB_LINK_UP: + break; + default: + panic("Bogus ntb_link_event %u\n", status); + } } /* Transport Init and teardown */ +static void +xeon_link_watchdog_hb(void *arg) +{ + struct ntb_transport_ctx *nt; + + nt = arg; + ntb_spad_write(nt->ntb, IF_NTB_WATCHDOG_SPAD, 0); + callout_reset(&nt->link_watchdog, 1 * hz, xeon_link_watchdog_hb, nt); +} + static int -ntb_transport_init(struct ntb_softc *ntb) +ntb_transport_probe(struct ntb_softc *ntb) { - struct ntb_netdev *nt = &net_softc; - int rc, i; + struct ntb_transport_ctx *nt = &net_softc; + struct ntb_transport_mw *mw; + uint64_t qp_bitmap; + int rc; + unsigned i; + + nt->mw_count = ntb_mw_count(ntb); + for (i = 0; i < nt->mw_count; i++) { + mw = &nt->mw_vec[i]; + + rc = ntb_mw_get_range(ntb, i, &mw->phys_addr, &mw->vbase, + &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size, + &mw->addr_limit); + if (rc != 0) + goto err; + + mw->buff_size = 0; + mw->xlat_size = 0; + mw->virt_addr = NULL; + mw->dma_addr = 0; + } + + qp_bitmap = ntb_db_valid_mask(ntb); + nt->qp_count = flsll(qp_bitmap); + KASSERT(nt->qp_count != 0, ("bogus db bitmap")); + nt->qp_count -= 1; + + if (max_num_clients != 0 && max_num_clients < nt->qp_count) + nt->qp_count = max_num_clients; + else if (nt->mw_count < nt->qp_count) + nt->qp_count = nt->mw_count; + KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count")); - nt->max_qps = max_num_clients; - ntb_register_transport(ntb, nt); mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF); mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF); - nt->qps = malloc(nt->max_qps * sizeof(struct ntb_transport_qp), - M_NTB_IF, M_WAITOK|M_ZERO); - - nt->qp_bitmap = ((uint64_t) 1 << nt->max_qps) - 1; + nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_IF, + M_WAITOK | M_ZERO); - for (i = 0; i < nt->max_qps; i++) + for (i = 0; i < nt->qp_count; i++) { + set_bit(i, &nt->qp_bitmap); + set_bit(i, &nt->qp_bitmap_free); ntb_transport_init_queue(nt, i); + } callout_init(&nt->link_work, 0); + callout_init(&nt->link_watchdog, 0); + TASK_INIT(&nt->link_cleanup, 0, ntb_transport_link_cleanup_work, nt); - rc = ntb_register_event_callback(ntb, - ntb_transport_event_callback); + rc = ntb_set_ctx(ntb, nt, &ntb_transport_ops); if (rc != 0) goto err; - if (ntb_query_link_status(ntb)) { - if (bootverbose) - device_printf(ntb_get_device(ntb), "link up\n"); - callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); - } - + nt->link_is_up = false; + ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); + ntb_link_event(ntb); + + callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt); + if (enable_xeon_watchdog != 0) + callout_reset(&nt->link_watchdog, 0, xeon_link_watchdog_hb, nt); return (0); err: - free(nt->qps, M_NTB_IF); - ntb_unregister_transport(ntb); + free(nt->qp_vec, M_NTB_IF); + nt->qp_vec = NULL; return (rc); } static void -ntb_transport_free(void *transport) +ntb_transport_free(struct ntb_transport_ctx *nt) { - struct ntb_netdev *nt = transport; struct ntb_softc *ntb = nt->ntb; - int i; - - nt->transport_link = NTB_LINK_DOWN; + struct _qpset qp_bitmap_alloc; + uint8_t i; + ntb_transport_link_cleanup(nt); + taskqueue_drain(taskqueue_swi, &nt->link_cleanup); callout_drain(&nt->link_work); + callout_drain(&nt->link_watchdog); - /* verify that all the qps are freed */ - for (i = 0; i < nt->max_qps; i++) - if (!test_bit(i, &nt->qp_bitmap)) - ntb_transport_free_queue(&nt->qps[i]); + BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc); + BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free); + /* Verify that all the QPs are freed */ + for (i = 0; i < nt->qp_count; i++) + if (test_bit(i, &qp_bitmap_alloc)) + ntb_transport_free_queue(&nt->qp_vec[i]); - ntb_unregister_event_callback(ntb); + ntb_link_disable(ntb); + ntb_clear_ctx(ntb); - for (i = 0; i < NTB_NUM_MW; i++) - if (nt->mw[i].virt_addr != NULL) - contigfree(nt->mw[i].virt_addr, nt->mw[i].size, - M_NTB_IF); + for (i = 0; i < nt->mw_count; i++) + ntb_free_mw(nt, i); - free(nt->qps, M_NTB_IF); - ntb_unregister_transport(ntb); + free(nt->qp_vec, M_NTB_IF); } static void -ntb_transport_init_queue(struct ntb_netdev *nt, unsigned int qp_num) +ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num) { + struct ntb_transport_mw *mw; struct ntb_transport_qp *qp; - unsigned int num_qps_mw, tx_size; - uint8_t mw_num = QP_TO_MW(qp_num); + vm_paddr_t mw_base; + uint64_t mw_size, qp_offset; + size_t tx_size; + unsigned num_qps_mw, mw_num, mw_count; + + mw_count = nt->mw_count; + mw_num = QP_TO_MW(nt, qp_num); + mw = &nt->mw_vec[mw_num]; - qp = &nt->qps[qp_num]; + qp = &nt->qp_vec[qp_num]; qp->qp_num = qp_num; qp->transport = nt; qp->ntb = nt->ntb; - qp->qp_link = NTB_LINK_DOWN; - qp->client_ready = NTB_LINK_DOWN; + qp->client_ready = false; qp->event_handler = NULL; + ntb_qp_link_down_reset(qp); - if (nt->max_qps % NTB_NUM_MW && mw_num < nt->max_qps % NTB_NUM_MW) - num_qps_mw = nt->max_qps / NTB_NUM_MW + 1; + if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count) + num_qps_mw = nt->qp_count / mw_count + 1; else - num_qps_mw = nt->max_qps / NTB_NUM_MW; + num_qps_mw = nt->qp_count / mw_count; + + mw_base = mw->phys_addr; + mw_size = mw->phys_size; + + tx_size = mw_size / num_qps_mw; + qp_offset = tx_size * (qp_num / mw_count); + + qp->tx_mw = mw->vbase + qp_offset; + KASSERT(qp->tx_mw != NULL, ("uh oh?")); + + /* XXX Assumes that a vm_paddr_t is equivalent to bus_addr_t */ + qp->tx_mw_phys = mw_base + qp_offset; + KASSERT(qp->tx_mw_phys != 0, ("uh oh?")); - tx_size = (unsigned int) ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw; - qp->rx_info = (struct ntb_rx_info *) - ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) + - (qp_num / NTB_NUM_MW * tx_size)); tx_size -= sizeof(struct ntb_rx_info); + qp->rx_info = (void *)(qp->tx_mw + tx_size); - qp->tx_mw = qp->rx_info + sizeof(struct ntb_rx_info); - qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header), - tx_size); + /* Due to house-keeping, there must be at least 2 buffs */ + qp->tx_max_frame = qmin(tx_size / 2, + transport_mtu + sizeof(struct ntb_payload_header)); qp->tx_max_entry = tx_size / qp->tx_max_frame; - qp->tx_index = 0; callout_init(&qp->link_work, 0); callout_init(&qp->queue_full, CALLOUT_MPSAFE); callout_init(&qp->rx_full, CALLOUT_MPSAFE); - mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN); - mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN); + mtx_init(&qp->ntb_rx_q_lock, "ntb rx q", NULL, MTX_SPIN); mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN); - TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp); + TASK_INIT(&qp->rx_completion_task, 0, ntb_complete_rxc, qp); + TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp); + STAILQ_INIT(&qp->rx_post_q); STAILQ_INIT(&qp->rx_pend_q); - STAILQ_INIT(&qp->rx_free_q); STAILQ_INIT(&qp->tx_free_q); + + callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); } static void @@ -578,18 +768,25 @@ ntb_transport_free_queue(struct ntb_tran callout_drain(&qp->link_work); - ntb_unregister_db_callback(qp->ntb, qp->qp_num); + ntb_db_set_mask(qp->ntb, 1ull << qp->qp_num); + taskqueue_drain(taskqueue_swi, &qp->rxc_db_work); + taskqueue_drain(taskqueue_swi, &qp->rx_completion_task); + + qp->cb_data = NULL; + qp->rx_handler = NULL; + qp->tx_handler = NULL; + qp->event_handler = NULL; - while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) + while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_pend_q))) free(entry, M_NTB_IF); - while ((entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q))) + while ((entry = ntb_list_rm(&qp->ntb_rx_q_lock, &qp->rx_post_q))) free(entry, M_NTB_IF); while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) free(entry, M_NTB_IF); - set_bit(qp->qp_num, &qp->transport->qp_bitmap); + set_bit(qp->qp_num, &qp->transport->qp_bitmap_free); } /** @@ -607,64 +804,48 @@ ntb_transport_free_queue(struct ntb_tran * RETURNS: pointer to newly created ntb_queue, NULL on error. */ static struct ntb_transport_qp * -ntb_transport_create_queue(void *data, struct ntb_softc *pdev, +ntb_transport_create_queue(void *data, struct ntb_softc *ntb, const struct ntb_queue_handlers *handlers) { struct ntb_queue_entry *entry; struct ntb_transport_qp *qp; - struct ntb_netdev *nt; + struct ntb_transport_ctx *nt; unsigned int free_queue; - int rc, i; + int i; - nt = ntb_find_transport(pdev); - if (nt == NULL) - goto err; + nt = ntb_get_ctx(ntb, NULL); + KASSERT(nt != NULL, ("bogus")); - free_queue = ffs(nt->qp_bitmap); + free_queue = ffs_bit(&nt->qp_bitmap); if (free_queue == 0) - goto err; + return (NULL); /* decrement free_queue to make it zero based */ free_queue--; - clear_bit(free_queue, &nt->qp_bitmap); - - qp = &nt->qps[free_queue]; + qp = &nt->qp_vec[free_queue]; + clear_bit(qp->qp_num, &nt->qp_bitmap_free); qp->cb_data = data; qp->rx_handler = handlers->rx_handler; qp->tx_handler = handlers->tx_handler; qp->event_handler = handlers->event_handler; for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { - entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, - M_WAITOK|M_ZERO); + entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO); entry->cb_data = nt->ifp; entry->buf = NULL; entry->len = transport_mtu; - ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q); + ntb_list_add(&qp->ntb_rx_q_lock, entry, &qp->rx_pend_q); } for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { - entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF, - M_WAITOK|M_ZERO); + entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO); ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); } - rc = ntb_register_db_callback(qp->ntb, free_queue, qp, - ntb_transport_rxc_db); - if (rc != 0) - goto err1; - + ntb_db_clear(ntb, 1ull << qp->qp_num); + ntb_db_clear_mask(ntb, 1ull << qp->qp_num); return (qp); - -err1: - while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q))) - free(entry, M_NTB_IF); - while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q))) - free(entry, M_NTB_IF); - set_bit(free_queue, &nt->qp_bitmap); -err: - return (NULL); } /** @@ -676,15 +857,17 @@ err: static void ntb_transport_link_up(struct ntb_transport_qp *qp) { + struct ntb_transport_ctx *nt; if (qp == NULL) return; - qp->client_ready = NTB_LINK_UP; - if (bootverbose) - device_printf(ntb_get_device(qp->ntb), "qp client ready\n"); + qp->client_ready = true; + + nt = qp->transport; + ntb_printf(2, "qp client ready\n"); - if (qp->transport->transport_link == NTB_LINK_UP) + if (qp->transport->link_is_up) callout_reset(&qp->link_work, 0, ntb_qp_link_work, qp); } @@ -700,7 +883,7 @@ ntb_transport_link_up(struct ntb_transpo * @len: length of the data buffer * * Enqueue a new transmit buffer onto the transport queue from which a NTB - * payload will be transmitted. This assumes that a lock is behing held to + * payload will be transmitted. This assumes that a lock is being held to * serialize access to the qp. * * RETURNS: An appropriate ERRNO error value on error, or zero for success. @@ -712,7 +895,7 @@ ntb_transport_tx_enqueue(struct ntb_tran struct ntb_queue_entry *entry; int rc; - if (qp == NULL || qp->qp_link != NTB_LINK_UP || len == 0) { + if (qp == NULL || !qp->link_is_up || len == 0) { CTR0(KTR_NTB, "TX: link not up"); return (EINVAL); } @@ -720,7 +903,8 @@ ntb_transport_tx_enqueue(struct ntb_tran entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q); if (entry == NULL) { CTR0(KTR_NTB, "TX: could not get entry from tx_free_q"); - return (ENOMEM); + qp->tx_err_no_buf++; + return (EBUSY); } CTR1(KTR_NTB, "TX: got entry %p from tx_free_q", entry); @@ -744,9 +928,9 @@ ntb_process_tx(struct ntb_transport_qp * { void *offset; - offset = (char *)qp->tx_mw + qp->tx_max_frame * qp->tx_index; + offset = qp->tx_mw + qp->tx_max_frame * qp->tx_index; CTR3(KTR_NTB, - "TX: process_tx: tx_pkts=%u, tx_index=%u, remote entry=%u", + "TX: process_tx: tx_pkts=%lu, tx_index=%u, remote entry=%u", qp->tx_pkts, qp->tx_index, qp->remote_rx_info->entry); if (qp->tx_index == qp->remote_rx_info->entry) { CTR0(KTR_NTB, "TX: ring full"); @@ -757,8 +941,11 @@ ntb_process_tx(struct ntb_transport_qp * if (entry->len > qp->tx_max_frame - sizeof(struct ntb_payload_header)) { if (qp->tx_handler != NULL) qp->tx_handler(qp, qp->cb_data, entry->buf, - EIO); + EIO); + else + m_freem(entry->buf); + entry->buf = NULL; ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q); CTR1(KTR_NTB, "TX: frame too big. returning entry %p to tx_free_q", @@ -766,7 +953,7 @@ ntb_process_tx(struct ntb_transport_qp * return (0); } CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset); - ntb_tx_copy_task(qp, entry, offset); + ntb_memcpy_tx(qp, entry, offset); qp->tx_index++; qp->tx_index %= qp->tx_max_entry; @@ -777,26 +964,37 @@ ntb_process_tx(struct ntb_transport_qp * } static void -ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, +ntb_memcpy_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry, void *offset) { struct ntb_payload_header *hdr; + /* This piece is from Linux' ntb_async_tx() */ + hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - + sizeof(struct ntb_payload_header)); + entry->x_hdr = hdr; + iowrite32(entry->len, &hdr->len); + iowrite32(qp->tx_pkts, &hdr->ver); + + /* This piece is ntb_memcpy_tx() */ CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset); - if (entry->buf != NULL) + if (entry->buf != NULL) { m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset); - hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame - - sizeof(struct ntb_payload_header)); - hdr->len = entry->len; /* TODO: replace with bus_space_write */ - hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */ - wmb(); - /* TODO: replace with bus_space_write */ - hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG; + /* + * Ensure that the data is fully copied before setting the + * flags + */ + wmb(); + } - ntb_ring_sdb(qp->ntb, qp->qp_num); + /* The rest is ntb_tx_copy_callback() */ + iowrite32(entry->flags | IF_NTB_DESC_DONE_FLAG, &hdr->flags); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***