From owner-svn-src-user@FreeBSD.ORG Wed Apr 18 19:11:21 2012 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id B2BF010656E3; Wed, 18 Apr 2012 19:11:21 +0000 (UTC) (envelope-from np@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 9CDA08FC12; Wed, 18 Apr 2012 19:11:21 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q3IJBLWS046717; Wed, 18 Apr 2012 19:11:21 GMT (envelope-from np@svn.freebsd.org) Received: (from np@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q3IJBLCU046712; Wed, 18 Apr 2012 19:11:21 GMT (envelope-from np@svn.freebsd.org) Message-Id: <201204181911.q3IJBLCU046712@svn.freebsd.org> From: Navdeep Parhar Date: Wed, 18 Apr 2012 19:11:21 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r234437 - in user/np/toe_iwarp/sys: contrib/rdma/krping modules/rdma/krping X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 18 Apr 2012 19:11:21 -0000 Author: np Date: Wed Apr 18 19:11:21 2012 New Revision: 234437 URL: http://svn.freebsd.org/changeset/base/234437 Log: - Switch krping to build against the new OFED code. - Add a "memlimit" option to krping that limits the memory region from where its buffers are allocated from. Obtained from: Chelsio Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c user/np/toe_iwarp/sys/modules/rdma/krping/Makefile Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Wed Apr 18 19:11:21 2012 (r234437) @@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include #include @@ -53,11 +52,13 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include -#include +#include +#include #include "getopt.h" #include "krping.h" @@ -83,6 +84,7 @@ static const struct krping_option krping {"bw", OPT_NOPARAM, 'B'}, {"tx-depth", OPT_INT, 't'}, {"poll", OPT_NOPARAM, 'P'}, + {"memlimit", OPT_INT, 'm'}, {NULL, 0, 0} }; @@ -254,10 +256,14 @@ static void krping_cq_event_handler(stru ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { if (wc.status) { - if (wc.status != IB_WC_WR_FLUSH_ERR) - log(LOG_ERR, "cq completion failed status %d\n", + if (wc.status == IB_WC_WR_FLUSH_ERR) { + DEBUG_LOG("cq flushed\n"); + continue; + } else { + log(LOG_CRIT, "cq completion failed status %d\n", wc.status); - goto error; + goto error; + } } switch (wc.opcode) { @@ -432,8 +438,17 @@ static int krping_setup_buffers(struct k } } - cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, - PAGE_SIZE, 0); + /* RNIC adapters have a limit upto which it can register physical memory + * If DMA-MR memory mode is set then normally driver registers maximum + * supported memory. After that if contigmalloc allocates memory beyond the + * specified RNIC limit then Krping may not work. + */ + if (cb->use_dmamr && cb->memlimit) + cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, cb->memlimit, + PAGE_SIZE, 0); + else + cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, + PAGE_SIZE, 0); if (!cb->rdma_buf) { log(LOG_ERR, "rdma_buf malloc failed\n"); @@ -458,8 +473,12 @@ static int krping_setup_buffers(struct k } if (!cb->server || cb->wlat || cb->rlat || cb->bw) { - cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, - 0, -1UL, PAGE_SIZE, 0); + if (cb->use_dmamr && cb->memlimit) + cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, + 0, cb->memlimit, PAGE_SIZE, 0); + else + cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, + 0, -1UL, PAGE_SIZE, 0); if (!cb->start_buf) { log(LOG_ERR, "start_buf malloc failed\n"); ret = ENOMEM; @@ -1636,6 +1655,8 @@ int krping_doit(char *cmd) cb->state = IDLE; cb->size = 64; cb->txdepth = RPING_SQ_DEPTH; + cb->use_dmamr = 1; + cb->memlimit = 0; mtx_init(&cb->lock, "krping mtx", NULL, MTX_DUPOK|MTX_DEF); while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, @@ -1713,6 +1734,15 @@ int krping_doit(char *cmd) case 'd': debug++; break; + case 'm': + cb->memlimit = optint; + if (cb->memlimit < 1) { + log(LOG_ERR, "Invalid memory limit %lu\n" + ,cb->memlimit); + ret = EINVAL; + } else + DEBUG_LOG(PFX "memory limit %d\n", (int)optint); + break; default: log(LOG_ERR, "unknown opt %s\n", optarg); ret = EINVAL; Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h Wed Apr 18 19:11:21 2012 (r234437) @@ -1,7 +1,7 @@ /* * $FreeBSD$ */ -#include +#include #include /* @@ -92,6 +92,8 @@ struct krping_cb { int count; /* ping count */ int size; /* ping data size */ int validate; /* validate ping data */ + uint64_t memlimit; /* limit of the physical memory that + can be registered with dma_mr mode */ /* CM stuff */ struct rdma_cm_id *cm_id; /* connection on client side,*/ Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c Wed Apr 18 19:11:21 2012 (r234437) @@ -14,7 +14,6 @@ __FBSDID("$FreeBSD$"); #include -#include #include /* uprintf */ #include #include /* defines used in kernel.h */ @@ -51,6 +50,9 @@ typedef struct s_krping { /* vars */ static struct cdev *krping_dev; +#undef MODULE_VERSION +#include + static int krping_loader(struct module *m, int what, void *arg) { @@ -175,6 +177,4 @@ krping_write(struct cdev *dev, struct ui return(err); } -MODULE_DEPEND(krping, rdma_core, 1, 1, 1); -MODULE_DEPEND(krping, rdma_cma, 1, 1, 1); DEV_MODULE(krping,krping_loader,NULL); Modified: user/np/toe_iwarp/sys/modules/rdma/krping/Makefile ============================================================================== --- user/np/toe_iwarp/sys/modules/rdma/krping/Makefile Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/modules/rdma/krping/Makefile Wed Apr 18 19:11:21 2012 (r234437) @@ -6,5 +6,7 @@ RDMA= ${.CURDIR}/../../../contrib/rdma/k KMOD= krping SRCS= krping.c krping_dev.c getopt.c SRCS+= bus_if.h device_if.h opt_sched.h pci_if.h pcib_if.h +SRCS+= vnode_if.h +CFLAGS+= -I${.CURDIR}/../../../ofed/include .include