Date: Thu, 15 Jun 2017 12:47:48 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r319974 - in projects/bsd_rdma_4_9/sys: contrib/rdma/krping modules/ibcore modules/ipoib modules/rdma/krping ofed/drivers/infiniband/core ofed/drivers/infiniband/debug ofed/drivers/infi... Message-ID: <201706151247.v5FClmMK066907@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Thu Jun 15 12:47:48 2017 New Revision: 319974 URL: https://svnweb.freebsd.org/changeset/base/319974 Log: Initial RoCE/infiniband kernel update to Linux v4.9. This patch currently supports: - ibcore as a kernel module only - krping as a kernel module only - ipoib as a kernel module only Sponsored by: Mellanox Technologies Added: projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_addr.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_cache.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_cq.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_iwpm_msg.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_iwpm_util.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_smi.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_sysfs.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_umem.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_umem_odp.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_umem_rbtree.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ib_verbs.c (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/iwpm_util.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/opa_smi.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_hdrs.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_umem_odp.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/iw_portmap.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/opa_port_info.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/opa_smi.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/rdma_vt.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/rdmavt_cq.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/rdmavt_mr.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/rdma/rdmavt_qp.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/uapi/ projects/bsd_rdma_4_9/sys/ofed/include/uapi/rdma/ projects/bsd_rdma_4_9/sys/ofed/include/uapi/rdma/mlx4-abi.h (contents, props changed) projects/bsd_rdma_4_9/sys/ofed/include/uapi/rdma/mlx5-abi.h (contents, props changed) Deleted: projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/addr.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/cache.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/peer_mem.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/smi.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/sysfs.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/umem.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/verbs.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/debug/Makefile projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/debug/memtrack.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/debug/memtrack.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/debug/mtrack.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/ulp/ipoib/Kconfig projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_peer_mem.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_user_verbs_exp.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_verbs_exp.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/peer_mem.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/sdp_socket.h Modified: projects/bsd_rdma_4_9/sys/contrib/rdma/krping/krping.c projects/bsd_rdma_4_9/sys/contrib/rdma/krping/krping.h projects/bsd_rdma_4_9/sys/contrib/rdma/krping/krping_dev.c projects/bsd_rdma_4_9/sys/modules/ibcore/Makefile projects/bsd_rdma_4_9/sys/modules/ipoib/Makefile projects/bsd_rdma_4_9/sys/modules/rdma/krping/Makefile projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/agent.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/agent.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/cm.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/cm_msgs.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/cma.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/core_priv.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/device.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/fmr_pool.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/iwcm.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/iwcm.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/mad.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/mad_priv.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/mad_rmpp.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/multicast.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/packer.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/sa_query.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/smi.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ucm.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ucma.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/ud_header.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/user_mad.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/uverbs.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/uverbs_cmd.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/uverbs_main.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/core/uverbs_marshall.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib.h projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_multicast.c projects/bsd_rdma_4_9/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_verbs.c projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_addr.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_cache.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_cm.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_mad.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_pack.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_pma.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_sa.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_smi.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_umem.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_user_mad.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_user_verbs.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/ib_verbs.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/iw_cm.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/rdma_cm.h projects/bsd_rdma_4_9/sys/ofed/include/rdma/rdma_user_cm.h Modified: projects/bsd_rdma_4_9/sys/contrib/rdma/krping/krping.c ============================================================================== --- projects/bsd_rdma_4_9/sys/contrib/rdma/krping/krping.c Thu Jun 15 12:18:46 2017 (r319973) +++ projects/bsd_rdma_4_9/sys/contrib/rdma/krping/krping.c Thu Jun 15 12:47:48 2017 (r319974) @@ -53,13 +53,14 @@ __FBSDID("$FreeBSD$"); #include "krping.h" #include "getopt.h" +#define PFX "krping: " + extern int krping_debug; -#define DEBUG_LOG(cb, x...) if (krping_debug) log(LOG_INFO, x) -#define PRINTF(cb, x...) log(LOG_INFO, x) +#define DEBUG_LOG(...) do { if (krping_debug) log(LOG_INFO, __VA_ARGS__); } while (0) #define BIND_INFO 1 MODULE_AUTHOR("Steve Wise"); -MODULE_DESCRIPTION("RDMA ping client/server"); +MODULE_DESCRIPTION("RDMA ping server"); MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(krping, 1); MODULE_DEPEND(krping, linuxkpi, 1, 1, 1); @@ -76,21 +77,19 @@ typedef uint64_t cycles_t; enum mem_type { DMA = 1, - FASTREG = 2, - MW = 3, - MR = 4 + REG = 2, }; static const struct krping_option krping_opts[] = { {"count", OPT_INT, 'C'}, {"size", OPT_INT, 'S'}, {"addr", OPT_STRING, 'a'}, + {"addr6", OPT_STRING, 'A'}, {"port", OPT_INT, 'p'}, {"verbose", OPT_NOPARAM, 'v'}, {"validate", OPT_NOPARAM, 'V'}, {"server", OPT_NOPARAM, 's'}, {"client", OPT_NOPARAM, 'c'}, - {"mem_mode", OPT_STRING, 'm'}, {"server_inv", OPT_NOPARAM, 'I'}, {"wlat", OPT_NOPARAM, 'l'}, {"rlat", OPT_NOPARAM, 'L'}, @@ -100,14 +99,14 @@ static const struct krping_option krping_opts[] = { {"poll", OPT_NOPARAM, 'P'}, {"local_dma_lkey", OPT_NOPARAM, 'Z'}, {"read_inv", OPT_NOPARAM, 'R'}, - {"fr", OPT_INT, 'f'}, + {"fr", OPT_NOPARAM, 'f'}, {NULL, 0, 0} }; #define htonll(x) cpu_to_be64((x)) #define ntohll(x) cpu_to_be64((x)) -static struct mutex krping_mutex; +static DEFINE_MUTEX(krping_mutex); /* * List of running krping threads. @@ -115,6 +114,13 @@ static struct mutex krping_mutex; static LIST_HEAD(krping_cbs); /* + * Invoke like this, one on each side, using the server's address on + * the RDMA device (iw%d): + * + * /bin/echo server,port=9999,addr=192.168.69.142,validate > /proc/krping + * /bin/echo client,port=9999,addr=192.168.69.142,validate > /proc/krping + * /bin/echo client,port=9999,addr6=2001:db8:0:f101::1,validate > /proc/krping + * * krping "ping/pong" loop: * client sends source rkey/addr/len * server receives source rkey/add/len @@ -163,42 +169,35 @@ struct krping_rdma_info { * Control block struct. */ struct krping_cb { - void *cookie; int server; /* 0 iff client */ struct ib_cq *cq; struct ib_pd *pd; struct ib_qp *qp; - enum mem_type mem; struct ib_mr *dma_mr; struct ib_fast_reg_page_list *page_list; int page_list_len; - struct ib_send_wr fastreg_wr; + struct ib_reg_wr reg_mr_wr; struct ib_send_wr invalidate_wr; - struct ib_mr *fastreg_mr; + struct ib_mr *reg_mr; int server_invalidate; int read_inv; u8 key; - struct ib_mw *mw; - struct ib_mw_bind bind_attr; - struct ib_recv_wr rq_wr; /* recv work request record */ struct ib_sge recv_sgl; /* recv single SGE */ - struct krping_rdma_info recv_buf;/* malloc'd buffer */ + struct krping_rdma_info recv_buf __aligned(16); /* malloc'd buffer */ u64 recv_dma_addr; DECLARE_PCI_UNMAP_ADDR(recv_mapping) - struct ib_mr *recv_mr; struct ib_send_wr sq_wr; /* send work requrest record */ struct ib_sge send_sgl; - struct krping_rdma_info send_buf;/* single send buf */ + struct krping_rdma_info send_buf __aligned(16); /* single send buf */ u64 send_dma_addr; DECLARE_PCI_UNMAP_ADDR(send_mapping) - struct ib_mr *send_mr; - struct ib_send_wr rdma_sq_wr; /* rdma work request record */ + struct ib_rdma_wr rdma_sq_wr; /* rdma work request record */ struct ib_sge rdma_sgl; /* rdma single SGE */ char *rdma_buf; /* used as rdma sink */ u64 rdma_dma_addr; @@ -219,8 +218,9 @@ struct krping_cb { struct krping_stats stats; uint16_t port; /* dst port in NBO */ - struct in_addr addr; /* dst addr in NBO */ + u8 addr[16]; /* dst addr in NBO */ char *addr_str; /* dst addr string */ + uint8_t addr_type; /* ADDR_FAMILY - IPv4/V6 */ int verbose; /* verbose logging */ int count; /* ping count */ int size; /* ping data size */ @@ -232,8 +232,7 @@ struct krping_cb { int poll; /* poll or block for rlat test */ int txdepth; /* SQ depth */ int local_dma_lkey; /* use 0 for lkey */ - int frtest; /* fastreg test */ - int testnum; + int frtest; /* reg test */ /* CM stuff */ struct rdma_cm_id *cm_id; /* connection on client side,*/ @@ -248,39 +247,34 @@ static int krping_cma_event_handler(struct rdma_cm_id int ret; struct krping_cb *cb = cma_id->context; - DEBUG_LOG(cb, "cma_event type %d cma_id %p (%s)\n", event->event, - cma_id, (cma_id == cb->cm_id) ? "parent" : "child"); + DEBUG_LOG("cma_event type %d cma_id %p (%s)\n", event->event, cma_id, + (cma_id == cb->cm_id) ? "parent" : "child"); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: cb->state = ADDR_RESOLVED; ret = rdma_resolve_route(cma_id, 2000); if (ret) { - PRINTF(cb, "rdma_resolve_route error %d\n", ret); + printk(KERN_ERR PFX "rdma_resolve_route error %d\n", + ret); wake_up_interruptible(&cb->sem); } break; case RDMA_CM_EVENT_ROUTE_RESOLVED: cb->state = ROUTE_RESOLVED; - cb->child_cm_id = cma_id; wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_CONNECT_REQUEST: - if (cb->state == IDLE) { - cb->state = CONNECT_REQUEST; - cb->child_cm_id = cma_id; - } else { - PRINTF(cb, "Received connection request in wrong state" - " (%d)\n", cb->state); - } - DEBUG_LOG(cb, "child cma %p\n", cb->child_cm_id); + cb->state = CONNECT_REQUEST; + cb->child_cm_id = cma_id; + DEBUG_LOG("child cma %p\n", cb->child_cm_id); wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_ESTABLISHED: - DEBUG_LOG(cb, "ESTABLISHED\n"); + DEBUG_LOG("ESTABLISHED\n"); if (!cb->server) { cb->state = CONNECTED; } @@ -292,24 +286,24 @@ static int krping_cma_event_handler(struct rdma_cm_id case RDMA_CM_EVENT_CONNECT_ERROR: case RDMA_CM_EVENT_UNREACHABLE: case RDMA_CM_EVENT_REJECTED: - PRINTF(cb, "cma event %d, error %d\n", event->event, + printk(KERN_ERR PFX "cma event %d, error %d\n", event->event, event->status); cb->state = ERROR; wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_DISCONNECTED: - PRINTF(cb, "DISCONNECT EVENT...\n"); + printk(KERN_ERR PFX "DISCONNECT EVENT...\n"); cb->state = ERROR; wake_up_interruptible(&cb->sem); break; case RDMA_CM_EVENT_DEVICE_REMOVAL: - PRINTF(cb, "cma detected device removal!!!!\n"); + printk(KERN_ERR PFX "cma detected device removal!!!!\n"); break; default: - PRINTF(cb, "oof bad type!\n"); + printk(KERN_ERR PFX "oof bad type!\n"); wake_up_interruptible(&cb->sem); break; } @@ -319,7 +313,7 @@ static int krping_cma_event_handler(struct rdma_cm_id static int server_recv(struct krping_cb *cb, struct ib_wc *wc) { if (wc->byte_len != sizeof(cb->recv_buf)) { - PRINTF(cb, "Received bogus data, size %d\n", + printk(KERN_ERR PFX "Received bogus data, size %d\n", wc->byte_len); return -1; } @@ -327,7 +321,7 @@ static int server_recv(struct krping_cb *cb, struct ib cb->remote_rkey = ntohl(cb->recv_buf.rkey); cb->remote_addr = ntohll(cb->recv_buf.buf); cb->remote_len = ntohl(cb->recv_buf.size); - DEBUG_LOG(cb, "Received rkey %x addr %llx len %d from peer\n", + DEBUG_LOG("Received rkey %x addr %llx len %d from peer\n", cb->remote_rkey, (unsigned long long)cb->remote_addr, cb->remote_len); @@ -342,7 +336,7 @@ static int server_recv(struct krping_cb *cb, struct ib static int client_recv(struct krping_cb *cb, struct ib_wc *wc) { if (wc->byte_len != sizeof(cb->recv_buf)) { - PRINTF(cb, "Received bogus data, size %d\n", + printk(KERN_ERR PFX "Received bogus data, size %d\n", wc->byte_len); return -1; } @@ -364,18 +358,22 @@ static void krping_cq_event_handler(struct ib_cq *cq, BUG_ON(cb->cq != cq); if (cb->state == ERROR) { - PRINTF(cb, "cq completion in ERROR state\n"); + printk(KERN_ERR PFX "cq completion in ERROR state\n"); return; } - if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) + if (cb->frtest) { + printk(KERN_ERR PFX "cq completion event in frtest!\n"); + return; + } + if (!cb->wlat && !cb->rlat && !cb->bw) ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { if (wc.status) { if (wc.status == IB_WC_WR_FLUSH_ERR) { - DEBUG_LOG(cb, "cq flushed\n"); + DEBUG_LOG("cq flushed\n"); continue; } else { - PRINTF(cb, "cq completion failed with " + printk(KERN_ERR PFX "cq completion failed with " "wr_id %jx status %d opcode %d vender_err %x\n", (uintmax_t)wc.wr_id, wc.status, wc.opcode, wc.vendor_err); goto error; @@ -384,44 +382,44 @@ static void krping_cq_event_handler(struct ib_cq *cq, switch (wc.opcode) { case IB_WC_SEND: - DEBUG_LOG(cb, "send completion\n"); + DEBUG_LOG("send completion\n"); cb->stats.send_bytes += cb->send_sgl.length; cb->stats.send_msgs++; break; case IB_WC_RDMA_WRITE: - DEBUG_LOG(cb, "rdma write completion\n"); - cb->stats.write_bytes += cb->rdma_sq_wr.sg_list->length; + DEBUG_LOG("rdma write completion\n"); + cb->stats.write_bytes += cb->rdma_sq_wr.wr.sg_list->length; cb->stats.write_msgs++; cb->state = RDMA_WRITE_COMPLETE; wake_up_interruptible(&cb->sem); break; case IB_WC_RDMA_READ: - DEBUG_LOG(cb, "rdma read completion\n"); - cb->stats.read_bytes += cb->rdma_sq_wr.sg_list->length; + DEBUG_LOG("rdma read completion\n"); + cb->stats.read_bytes += cb->rdma_sq_wr.wr.sg_list->length; cb->stats.read_msgs++; cb->state = RDMA_READ_COMPLETE; wake_up_interruptible(&cb->sem); break; case IB_WC_RECV: - DEBUG_LOG(cb, "recv completion\n"); + DEBUG_LOG("recv completion\n"); cb->stats.recv_bytes += sizeof(cb->recv_buf); cb->stats.recv_msgs++; - if (cb->wlat || cb->rlat || cb->bw || cb->frtest) + if (cb->wlat || cb->rlat || cb->bw) ret = server_recv(cb, &wc); else ret = cb->server ? server_recv(cb, &wc) : client_recv(cb, &wc); if (ret) { - PRINTF(cb, "recv wc error: %d\n", ret); + printk(KERN_ERR PFX "recv wc error: %d\n", ret); goto error; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { - PRINTF(cb, "post recv error: %d\n", + printk(KERN_ERR PFX "post recv error: %d\n", ret); goto error; } @@ -429,14 +427,14 @@ static void krping_cq_event_handler(struct ib_cq *cq, break; default: - PRINTF(cb, + printk(KERN_ERR PFX "%s:%d Unexpected opcode %d, Shutting down\n", __func__, __LINE__, wc.opcode); goto error; } } if (ret) { - PRINTF(cb, "poll error %d\n", ret); + printk(KERN_ERR PFX "poll error %d\n", ret); goto error; } return; @@ -450,7 +448,7 @@ static int krping_accept(struct krping_cb *cb) struct rdma_conn_param conn_param; int ret; - DEBUG_LOG(cb, "accepting client connection request\n"); + DEBUG_LOG("accepting client connection request\n"); memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = 1; @@ -458,14 +456,14 @@ static int krping_accept(struct krping_cb *cb) ret = rdma_accept(cb->child_cm_id, &conn_param); if (ret) { - PRINTF(cb, "rdma_accept error: %d\n", ret); + printk(KERN_ERR PFX "rdma_accept error: %d\n", ret); return ret; } - if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { + if (!cb->wlat && !cb->rlat && !cb->bw) { wait_event_interruptible(cb->sem, cb->state >= CONNECTED); if (cb->state == ERROR) { - PRINTF(cb, "wait for CONNECTED state %d\n", + printk(KERN_ERR PFX "wait for CONNECTED state %d\n", cb->state); return -1; } @@ -477,278 +475,120 @@ static void krping_setup_wr(struct krping_cb *cb) { cb->recv_sgl.addr = cb->recv_dma_addr; cb->recv_sgl.length = sizeof cb->recv_buf; - if (cb->local_dma_lkey) - cb->recv_sgl.lkey = cb->qp->device->local_dma_lkey; - else if (cb->mem == DMA) - cb->recv_sgl.lkey = cb->dma_mr->lkey; - else - cb->recv_sgl.lkey = cb->recv_mr->lkey; + cb->recv_sgl.lkey = cb->pd->local_dma_lkey; cb->rq_wr.sg_list = &cb->recv_sgl; cb->rq_wr.num_sge = 1; cb->send_sgl.addr = cb->send_dma_addr; cb->send_sgl.length = sizeof cb->send_buf; - if (cb->local_dma_lkey) - cb->send_sgl.lkey = cb->qp->device->local_dma_lkey; - else if (cb->mem == DMA) - cb->send_sgl.lkey = cb->dma_mr->lkey; - else - cb->send_sgl.lkey = cb->send_mr->lkey; + cb->send_sgl.lkey = cb->pd->local_dma_lkey; cb->sq_wr.opcode = IB_WR_SEND; cb->sq_wr.send_flags = IB_SEND_SIGNALED; cb->sq_wr.sg_list = &cb->send_sgl; cb->sq_wr.num_sge = 1; - if (cb->server || cb->wlat || cb->rlat || cb->bw || cb->frtest) { + if (cb->server || cb->wlat || cb->rlat || cb->bw) { cb->rdma_sgl.addr = cb->rdma_dma_addr; - if (cb->mem == MR) - cb->rdma_sgl.lkey = cb->rdma_mr->lkey; - cb->rdma_sq_wr.send_flags = IB_SEND_SIGNALED; - cb->rdma_sq_wr.sg_list = &cb->rdma_sgl; - cb->rdma_sq_wr.num_sge = 1; + cb->rdma_sq_wr.wr.send_flags = IB_SEND_SIGNALED; + cb->rdma_sq_wr.wr.sg_list = &cb->rdma_sgl; + cb->rdma_sq_wr.wr.num_sge = 1; } - switch(cb->mem) { - case FASTREG: + /* + * A chain of 2 WRs, INVALDATE_MR + REG_MR. + * both unsignaled. The client uses them to reregister + * the rdma buffers with a new key each iteration. + */ + cb->reg_mr_wr.wr.opcode = IB_WR_REG_MR; + cb->reg_mr_wr.mr = cb->reg_mr; - /* - * A chain of 2 WRs, INVALDATE_MR + FAST_REG_MR. - * both unsignaled. The client uses them to reregister - * the rdma buffers with a new key each iteration. - */ - cb->fastreg_wr.opcode = IB_WR_FAST_REG_MR; - cb->fastreg_wr.wr.fast_reg.page_shift = PAGE_SHIFT; - cb->fastreg_wr.wr.fast_reg.length = cb->size; - cb->fastreg_wr.wr.fast_reg.page_list = cb->page_list; - cb->fastreg_wr.wr.fast_reg.page_list_len = cb->page_list_len; - - cb->invalidate_wr.next = &cb->fastreg_wr; - cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; - break; - case MW: - cb->bind_attr.wr_id = 0xabbaabba; - cb->bind_attr.send_flags = 0; /* unsignaled */ -#ifdef BIND_INFO - cb->bind_attr.bind_info.length = cb->size; -#else - cb->bind_attr.length = cb->size; -#endif - break; - default: - break; - } + cb->invalidate_wr.next = &cb->reg_mr_wr.wr; + cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; } static int krping_setup_buffers(struct krping_cb *cb) { int ret; - struct ib_phys_buf buf; - u64 iovbase; - DEBUG_LOG(cb, "krping_setup_buffers called on cb %p\n", cb); + DEBUG_LOG(PFX "krping_setup_buffers called on cb %p\n", cb); - cb->recv_dma_addr = ib_dma_map_single(cb->pd->device, + cb->recv_dma_addr = ib_dma_map_single(cb->pd->device, &cb->recv_buf, sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); - cb->send_dma_addr = ib_dma_map_single(cb->pd->device, + cb->send_dma_addr = ib_dma_map_single(cb->pd->device, &cb->send_buf, sizeof(cb->send_buf), DMA_BIDIRECTIONAL); pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); - if (cb->mem == DMA) { - cb->dma_mr = ib_get_dma_mr(cb->pd, IB_ACCESS_LOCAL_WRITE| - IB_ACCESS_REMOTE_READ| - IB_ACCESS_REMOTE_WRITE); - if (IS_ERR(cb->dma_mr)) { - DEBUG_LOG(cb, "reg_dmamr failed\n"); - ret = PTR_ERR(cb->dma_mr); - goto bail; - } - } else { - if (!cb->local_dma_lkey) { - buf.addr = cb->recv_dma_addr; - buf.size = sizeof cb->recv_buf; - DEBUG_LOG(cb, "recv buf dma_addr %jx size %d\n", - (uintmax_t)buf.addr, (int)buf.size); - iovbase = cb->recv_dma_addr; - cb->recv_mr = ib_reg_phys_mr(cb->pd, &buf, 1, - IB_ACCESS_LOCAL_WRITE, - &iovbase); - - if (IS_ERR(cb->recv_mr)) { - DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); - ret = PTR_ERR(cb->recv_mr); - goto bail; - } - - buf.addr = cb->send_dma_addr; - buf.size = sizeof cb->send_buf; - DEBUG_LOG(cb, "send buf dma_addr %jx size %d\n", - (uintmax_t)buf.addr, (int)buf.size); - iovbase = cb->send_dma_addr; - cb->send_mr = ib_reg_phys_mr(cb->pd, &buf, 1, - 0, &iovbase); - - if (IS_ERR(cb->send_mr)) { - DEBUG_LOG(cb, "send_buf reg_mr failed\n"); - ret = PTR_ERR(cb->send_mr); - goto bail; - } - } - } - - cb->rdma_buf = kmalloc(cb->size, GFP_KERNEL); + cb->rdma_buf = ib_dma_alloc_coherent(cb->pd->device, cb->size, + &cb->rdma_dma_addr, + GFP_KERNEL); if (!cb->rdma_buf) { - DEBUG_LOG(cb, "rdma_buf malloc failed\n"); + DEBUG_LOG(PFX "rdma_buf allocation failed\n"); ret = -ENOMEM; goto bail; } - - cb->rdma_dma_addr = ib_dma_map_single(cb->pd->device, - cb->rdma_buf, cb->size, - DMA_BIDIRECTIONAL); pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); - if (cb->mem != DMA) { - switch (cb->mem) { - case FASTREG: - cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + - PAGE_SIZE) >> PAGE_SHIFT; - cb->page_list = ib_alloc_fast_reg_page_list( - cb->pd->device, - cb->page_list_len); - if (IS_ERR(cb->page_list)) { - DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); - ret = PTR_ERR(cb->page_list); - goto bail; - } - cb->fastreg_mr = ib_alloc_fast_reg_mr(cb->pd, - cb->page_list->max_page_list_len); - if (IS_ERR(cb->fastreg_mr)) { - DEBUG_LOG(cb, "recv_buf reg_mr failed\n"); - ret = PTR_ERR(cb->fastreg_mr); - goto bail; - } - DEBUG_LOG(cb, "fastreg rkey 0x%x page_list %p" - " page_list_len %u\n", cb->fastreg_mr->rkey, - cb->page_list, cb->page_list_len); - break; - case MW: - cb->mw = ib_alloc_mw(cb->pd,IB_MW_TYPE_1); - if (IS_ERR(cb->mw)) { - DEBUG_LOG(cb, "recv_buf alloc_mw failed\n"); - ret = PTR_ERR(cb->mw); - goto bail; - } - DEBUG_LOG(cb, "mw rkey 0x%x\n", cb->mw->rkey); - /*FALLTHROUGH*/ - case MR: - buf.addr = cb->rdma_dma_addr; - buf.size = cb->size; - iovbase = cb->rdma_dma_addr; - cb->rdma_mr = ib_reg_phys_mr(cb->pd, &buf, 1, - IB_ACCESS_LOCAL_WRITE| - IB_ACCESS_REMOTE_READ| - IB_ACCESS_REMOTE_WRITE, - &iovbase); - if (IS_ERR(cb->rdma_mr)) { - DEBUG_LOG(cb, "rdma_buf reg_mr failed\n"); - ret = PTR_ERR(cb->rdma_mr); - goto bail; - } - DEBUG_LOG(cb, "rdma buf dma_addr %jx size %d mr rkey 0x%x\n", - (uintmax_t)buf.addr, (int)buf.size, cb->rdma_mr->rkey); - break; - default: - ret = -EINVAL; - goto bail; - break; - } + cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + PAGE_SIZE) + >> PAGE_SHIFT; + cb->reg_mr = ib_alloc_mr(cb->pd, IB_MR_TYPE_MEM_REG, + cb->page_list_len); + if (IS_ERR(cb->reg_mr)) { + ret = PTR_ERR(cb->reg_mr); + DEBUG_LOG(PFX "recv_buf reg_mr failed %d\n", ret); + goto bail; } + DEBUG_LOG(PFX "reg rkey 0x%x page_list_len %u\n", + cb->reg_mr->rkey, cb->page_list_len); - if (!cb->server || cb->wlat || cb->rlat || cb->bw || cb->frtest) { + if (!cb->server || cb->wlat || cb->rlat || cb->bw) { - cb->start_buf = kmalloc(cb->size, GFP_KERNEL); + cb->start_buf = ib_dma_alloc_coherent(cb->pd->device, cb->size, + &cb->start_dma_addr, + GFP_KERNEL); if (!cb->start_buf) { - DEBUG_LOG(cb, "start_buf malloc failed\n"); + DEBUG_LOG(PFX "start_buf malloc failed\n"); ret = -ENOMEM; goto bail; } - - cb->start_dma_addr = ib_dma_map_single(cb->pd->device, - cb->start_buf, cb->size, - DMA_BIDIRECTIONAL); pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); - - if (cb->mem == MR || cb->mem == MW) { - unsigned flags = IB_ACCESS_REMOTE_READ; - - if (cb->wlat || cb->rlat || cb->bw || cb->frtest) { - flags |= IB_ACCESS_LOCAL_WRITE | - IB_ACCESS_REMOTE_WRITE; - } - - buf.addr = cb->start_dma_addr; - buf.size = cb->size; - DEBUG_LOG(cb, "start buf dma_addr %jx size %d\n", - (uintmax_t)buf.addr, (int)buf.size); - iovbase = cb->start_dma_addr; - cb->start_mr = ib_reg_phys_mr(cb->pd, &buf, 1, - flags, - &iovbase); - - if (IS_ERR(cb->start_mr)) { - DEBUG_LOG(cb, "start_buf reg_mr failed\n"); - ret = PTR_ERR(cb->start_mr); - goto bail; - } - } } krping_setup_wr(cb); - DEBUG_LOG(cb, "allocated & registered buffers...\n"); + DEBUG_LOG(PFX "allocated & registered buffers...\n"); return 0; bail: - if (cb->fastreg_mr && !IS_ERR(cb->fastreg_mr)) - ib_dereg_mr(cb->fastreg_mr); - if (cb->mw && !IS_ERR(cb->mw)) - ib_dealloc_mw(cb->mw); + if (cb->reg_mr && !IS_ERR(cb->reg_mr)) + ib_dereg_mr(cb->reg_mr); if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) ib_dereg_mr(cb->rdma_mr); - if (cb->page_list && !IS_ERR(cb->page_list)) - ib_free_fast_reg_page_list(cb->page_list); if (cb->dma_mr && !IS_ERR(cb->dma_mr)) ib_dereg_mr(cb->dma_mr); - if (cb->recv_mr && !IS_ERR(cb->recv_mr)) - ib_dereg_mr(cb->recv_mr); - if (cb->send_mr && !IS_ERR(cb->send_mr)) - ib_dereg_mr(cb->send_mr); - if (cb->rdma_buf) - kfree(cb->rdma_buf); - if (cb->start_buf) - kfree(cb->start_buf); + if (cb->rdma_buf) { + ib_dma_free_coherent(cb->pd->device, cb->size, cb->rdma_buf, + cb->rdma_dma_addr); + } + if (cb->start_buf) { + ib_dma_free_coherent(cb->pd->device, cb->size, cb->start_buf, + cb->start_dma_addr); + } return ret; } static void krping_free_buffers(struct krping_cb *cb) { - DEBUG_LOG(cb, "krping_free_buffers called on cb %p\n", cb); + DEBUG_LOG("krping_free_buffers called on cb %p\n", cb); if (cb->dma_mr) ib_dereg_mr(cb->dma_mr); - if (cb->send_mr) - ib_dereg_mr(cb->send_mr); - if (cb->recv_mr) - ib_dereg_mr(cb->recv_mr); if (cb->rdma_mr) ib_dereg_mr(cb->rdma_mr); if (cb->start_mr) ib_dereg_mr(cb->start_mr); - if (cb->fastreg_mr) - ib_dereg_mr(cb->fastreg_mr); - if (cb->mw) - ib_dealloc_mw(cb->mw); + if (cb->reg_mr) + ib_dereg_mr(cb->reg_mr); dma_unmap_single(cb->pd->device->dma_device, pci_unmap_addr(cb, recv_mapping), @@ -756,15 +596,13 @@ static void krping_free_buffers(struct krping_cb *cb) dma_unmap_single(cb->pd->device->dma_device, pci_unmap_addr(cb, send_mapping), sizeof(cb->send_buf), DMA_BIDIRECTIONAL); - dma_unmap_single(cb->pd->device->dma_device, - pci_unmap_addr(cb, rdma_mapping), - cb->size, DMA_BIDIRECTIONAL); - kfree(cb->rdma_buf); + + ib_dma_free_coherent(cb->pd->device, cb->size, cb->rdma_buf, + cb->rdma_dma_addr); + if (cb->start_buf) { - dma_unmap_single(cb->pd->device->dma_device, - pci_unmap_addr(cb, start_mapping), - cb->size, DMA_BIDIRECTIONAL); - kfree(cb->start_buf); + ib_dma_free_coherent(cb->pd->device, cb->size, cb->start_buf, + cb->start_dma_addr); } } @@ -776,6 +614,11 @@ static int krping_create_qp(struct krping_cb *cb) memset(&init_attr, 0, sizeof(init_attr)); init_attr.cap.max_send_wr = cb->txdepth; init_attr.cap.max_recv_wr = 2; + + /* For flush_qp() */ + init_attr.cap.max_send_wr++; + init_attr.cap.max_recv_wr++; + init_attr.cap.max_recv_sge = 1; init_attr.cap.max_send_sge = 1; init_attr.qp_type = IB_QPT_RC; @@ -806,38 +649,42 @@ static void krping_free_qp(struct krping_cb *cb) static int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) { int ret; - cb->pd = ib_alloc_pd(cm_id->device); + struct ib_cq_init_attr attr = {0}; + + cb->pd = ib_alloc_pd(cm_id->device, 0); if (IS_ERR(cb->pd)) { - PRINTF(cb, "ib_alloc_pd failed\n"); + printk(KERN_ERR PFX "ib_alloc_pd failed\n"); return PTR_ERR(cb->pd); } - DEBUG_LOG(cb, "created pd %p\n", cb->pd); + DEBUG_LOG("created pd %p\n", cb->pd); strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); + attr.cqe = cb->txdepth * 2; + attr.comp_vector = 0; cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, - cb, cb->txdepth * 2, 0); + cb, &attr); if (IS_ERR(cb->cq)) { - PRINTF(cb, "ib_create_cq failed\n"); + printk(KERN_ERR PFX "ib_create_cq failed\n"); ret = PTR_ERR(cb->cq); goto err1; } - DEBUG_LOG(cb, "created cq %p\n", cb->cq); + DEBUG_LOG("created cq %p\n", cb->cq); if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); if (ret) { - PRINTF(cb, "ib_create_cq failed\n"); + printk(KERN_ERR PFX "ib_create_cq failed\n"); goto err2; } } ret = krping_create_qp(cb); if (ret) { - PRINTF(cb, "krping_create_qp failed: %d\n", ret); + printk(KERN_ERR PFX "krping_create_qp failed: %d\n", ret); goto err2; } - DEBUG_LOG(cb, "created qp %p\n", cb->qp); + DEBUG_LOG("created qp %p\n", cb->qp); return 0; err2: ib_destroy_cq(cb->cq); @@ -848,115 +695,54 @@ err1: /* * return the (possibly rebound) rkey for the rdma buffer. - * FASTREG mode: invalidate and rebind via fastreg wr. - * MW mode: rebind the MW. + * REG mode: invalidate and rebind via reg wr. * other modes: just return the mr rkey. */ static u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) { - u32 rkey = 0xffffffff; - u64 p; + u32 rkey; struct ib_send_wr *bad_wr; - int i; int ret; + struct scatterlist sg = {0}; - switch (cb->mem) { - case FASTREG: - cb->invalidate_wr.ex.invalidate_rkey = cb->fastreg_mr->rkey; + cb->invalidate_wr.ex.invalidate_rkey = cb->reg_mr->rkey; - /* - * Update the fastreg key. - */ - ib_update_fast_reg_key(cb->fastreg_mr, ++cb->key); - cb->fastreg_wr.wr.fast_reg.rkey = cb->fastreg_mr->rkey; + /* + * Update the reg key. + */ + ib_update_fast_reg_key(cb->reg_mr, ++cb->key); + cb->reg_mr_wr.key = cb->reg_mr->rkey; - /* - * Update the fastreg WR with new buf info. - */ - if (buf == (u64)cb->start_dma_addr) - cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_READ; - else - cb->fastreg_wr.wr.fast_reg.access_flags = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; - cb->fastreg_wr.wr.fast_reg.iova_start = buf; - p = (u64)(buf & PAGE_MASK); - for (i=0; i < cb->fastreg_wr.wr.fast_reg.page_list_len; - i++, p += PAGE_SIZE) { - cb->page_list->page_list[i] = p; - DEBUG_LOG(cb, "page_list[%d] 0x%jx\n", i, (uintmax_t)p); - } + /* + * Update the reg WR with new buf info. + */ + if (buf == (u64)cb->start_dma_addr) + cb->reg_mr_wr.access = IB_ACCESS_REMOTE_READ; + else + cb->reg_mr_wr.access = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; + sg_dma_address(&sg) = buf; + sg_dma_len(&sg) = cb->size; - DEBUG_LOG(cb, "post_inv = %d, fastreg new rkey 0x%x shift %u len %u" - " iova_start %jx page_list_len %u\n", - post_inv, - cb->fastreg_wr.wr.fast_reg.rkey, - cb->fastreg_wr.wr.fast_reg.page_shift, - (unsigned)cb->fastreg_wr.wr.fast_reg.length, - (uintmax_t)cb->fastreg_wr.wr.fast_reg.iova_start, - cb->fastreg_wr.wr.fast_reg.page_list_len); + ret = ib_map_mr_sg(cb->reg_mr, &sg, 1, NULL, PAGE_SIZE); + BUG_ON(ret <= 0 || ret > cb->page_list_len); - if (post_inv) - ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); - else - ret = ib_post_send(cb->qp, &cb->fastreg_wr, &bad_wr); - if (ret) { - PRINTF(cb, "post send error %d\n", ret); - cb->state = ERROR; - } - rkey = cb->fastreg_mr->rkey; - break; - case MW: - /* - * Update the MW with new buf info. - */ - if (buf == (u64)cb->start_dma_addr) { -#ifdef BIND_INFO - cb->bind_attr.bind_info.mw_access_flags = IB_ACCESS_REMOTE_READ; - cb->bind_attr.bind_info.mr = cb->start_mr; -#else - cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_READ; - cb->bind_attr.mr = cb->start_mr; -#endif - } else { -#ifdef BIND_INFO - cb->bind_attr.bind_info.mw_access_flags = IB_ACCESS_REMOTE_WRITE; - cb->bind_attr.bind_info.mr = cb->rdma_mr; -#else - cb->bind_attr.mw_access_flags = IB_ACCESS_REMOTE_WRITE; - cb->bind_attr.mr = cb->rdma_mr; -#endif - } -#ifdef BIND_INFO - cb->bind_attr.bind_info.addr = buf; -#else - cb->bind_attr.addr = buf; -#endif - DEBUG_LOG(cb, "binding mw rkey 0x%x to buf %jx mr rkey 0x%x\n", -#ifdef BIND_INFO - cb->mw->rkey, (uintmax_t)buf, cb->bind_attr.bind_info.mr->rkey); -#else - cb->mw->rkey, buf, cb->bind_attr.mr->rkey); -#endif - ret = ib_bind_mw(cb->qp, cb->mw, &cb->bind_attr); - if (ret) { - PRINTF(cb, "bind mw error %d\n", ret); - cb->state = ERROR; - } else - rkey = cb->mw->rkey; - break; - case MR: - if (buf == (u64)cb->start_dma_addr) - rkey = cb->start_mr->rkey; - else - rkey = cb->rdma_mr->rkey; - break; - case DMA: - rkey = cb->dma_mr->rkey; - break; - default: - PRINTF(cb, "%s:%d case ERROR\n", __func__, __LINE__); + DEBUG_LOG(PFX "post_inv = %d, reg_mr new rkey 0x%x pgsz %u len %u" + " iova_start %llx\n", + post_inv, + cb->reg_mr_wr.key, + cb->reg_mr->page_size, + cb->reg_mr->length, + (unsigned long long)cb->reg_mr->iova); + + if (post_inv) + ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); + else + ret = ib_post_send(cb->qp, &cb->reg_mr_wr.wr, &bad_wr); + if (ret) { + printk(KERN_ERR PFX "post send error %d\n", ret); cb->state = ERROR; - break; } + rkey = cb->reg_mr->rkey; return rkey; } @@ -966,16 +752,16 @@ static void krping_format_send(struct krping_cb *cb, u u32 rkey; /* - * Client side will do fastreg or mw bind before + * Client side will do reg or mw bind before * advertising the rdma buffer. Server side * sends have no data. */ - if (!cb->server || cb->wlat || cb->rlat || cb->bw || cb->frtest) { + if (!cb->server || cb->wlat || cb->rlat || cb->bw) { rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); info->buf = htonll(buf); info->rkey = htonl(rkey); info->size = htonl(cb->size); - DEBUG_LOG(cb, "RDMA addr %llx rkey %x len %d\n", + DEBUG_LOG("RDMA addr %llx rkey %x len %d\n", (unsigned long long)buf, rkey, cb->size); } } @@ -989,111 +775,102 @@ static void krping_test_server(struct krping_cb *cb) /* Wait for client's Start STAG/TO/Len */ wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); if (cb->state != RDMA_READ_ADV) { - PRINTF(cb, "wait for RDMA_READ_ADV state %d\n", + printk(KERN_ERR PFX "wait for RDMA_READ_ADV state %d\n", cb->state); break; } - DEBUG_LOG(cb, "server received sink adv\n"); + DEBUG_LOG("server received sink adv\n"); - cb->rdma_sq_wr.wr.rdma.rkey = cb->remote_rkey; - cb->rdma_sq_wr.wr.rdma.remote_addr = cb->remote_addr; - cb->rdma_sq_wr.sg_list->length = cb->remote_len; - cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 1); + cb->rdma_sq_wr.rkey = cb->remote_rkey; + cb->rdma_sq_wr.remote_addr = cb->remote_addr; + cb->rdma_sq_wr.wr.sg_list->length = cb->remote_len; + cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, !cb->read_inv); + cb->rdma_sq_wr.wr.next = NULL; /* Issue RDMA Read. */ if (cb->read_inv) - cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ_WITH_INV; + cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV; else { - cb->rdma_sq_wr.opcode = IB_WR_RDMA_READ; - if (cb->mem == FASTREG) { - /* - * Immediately follow the read with a - * fenced LOCAL_INV. - */ - cb->rdma_sq_wr.next = &inv; - memset(&inv, 0, sizeof inv); - inv.opcode = IB_WR_LOCAL_INV; - inv.ex.invalidate_rkey = cb->fastreg_mr->rkey; - inv.send_flags = IB_SEND_FENCE; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201706151247.v5FClmMK066907>