Date: Wed, 18 Apr 2012 19:11:21 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r234437 - in user/np/toe_iwarp/sys: contrib/rdma/krping modules/rdma/krping Message-ID: <201204181911.q3IJBLCU046712@svn.freebsd.org>
index | next in thread | raw e-mail
Author: np Date: Wed Apr 18 19:11:21 2012 New Revision: 234437 URL: http://svn.freebsd.org/changeset/base/234437 Log: - Switch krping to build against the new OFED code. - Add a "memlimit" option to krping that limits the memory region from where its buffers are allocated from. Obtained from: Chelsio Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c user/np/toe_iwarp/sys/modules/rdma/krping/Makefile Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping.c Wed Apr 18 19:11:21 2012 (r234437) @@ -41,7 +41,6 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/socket.h> -#include <sys/module.h> #include <sys/endian.h> #include <sys/limits.h> #include <sys/proc.h> @@ -53,11 +52,13 @@ __FBSDID("$FreeBSD$"); #include <sys/queue.h> #include <sys/taskqueue.h> #include <sys/syslog.h> +#include <netinet/in.h> #include <vm/vm.h> #include <vm/pmap.h> -#include <contrib/rdma/rdma_cm.h> +#include <linux/types.h> +#include <rdma/rdma_cm.h> #include "getopt.h" #include "krping.h" @@ -83,6 +84,7 @@ static const struct krping_option krping {"bw", OPT_NOPARAM, 'B'}, {"tx-depth", OPT_INT, 't'}, {"poll", OPT_NOPARAM, 'P'}, + {"memlimit", OPT_INT, 'm'}, {NULL, 0, 0} }; @@ -254,10 +256,14 @@ static void krping_cq_event_handler(stru ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { if (wc.status) { - if (wc.status != IB_WC_WR_FLUSH_ERR) - log(LOG_ERR, "cq completion failed status %d\n", + if (wc.status == IB_WC_WR_FLUSH_ERR) { + DEBUG_LOG("cq flushed\n"); + continue; + } else { + log(LOG_CRIT, "cq completion failed status %d\n", wc.status); - goto error; + goto error; + } } switch (wc.opcode) { @@ -432,8 +438,17 @@ static int krping_setup_buffers(struct k } } - cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, - PAGE_SIZE, 0); + /* RNIC adapters have a limit upto which it can register physical memory + * If DMA-MR memory mode is set then normally driver registers maximum + * supported memory. After that if contigmalloc allocates memory beyond the + * specified RNIC limit then Krping may not work. + */ + if (cb->use_dmamr && cb->memlimit) + cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, cb->memlimit, + PAGE_SIZE, 0); + else + cb->rdma_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, 0, -1UL, + PAGE_SIZE, 0); if (!cb->rdma_buf) { log(LOG_ERR, "rdma_buf malloc failed\n"); @@ -458,8 +473,12 @@ static int krping_setup_buffers(struct k } if (!cb->server || cb->wlat || cb->rlat || cb->bw) { - cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, - 0, -1UL, PAGE_SIZE, 0); + if (cb->use_dmamr && cb->memlimit) + cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, + 0, cb->memlimit, PAGE_SIZE, 0); + else + cb->start_buf = contigmalloc(cb->size, M_DEVBUF, M_WAITOK, + 0, -1UL, PAGE_SIZE, 0); if (!cb->start_buf) { log(LOG_ERR, "start_buf malloc failed\n"); ret = ENOMEM; @@ -1636,6 +1655,8 @@ int krping_doit(char *cmd) cb->state = IDLE; cb->size = 64; cb->txdepth = RPING_SQ_DEPTH; + cb->use_dmamr = 1; + cb->memlimit = 0; mtx_init(&cb->lock, "krping mtx", NULL, MTX_DUPOK|MTX_DEF); while ((op = krping_getopt("krping", &cmd, krping_opts, NULL, &optarg, @@ -1713,6 +1734,15 @@ int krping_doit(char *cmd) case 'd': debug++; break; + case 'm': + cb->memlimit = optint; + if (cb->memlimit < 1) { + log(LOG_ERR, "Invalid memory limit %lu\n" + ,cb->memlimit); + ret = EINVAL; + } else + DEBUG_LOG(PFX "memory limit %d\n", (int)optint); + break; default: log(LOG_ERR, "unknown opt %s\n", optarg); ret = EINVAL; Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping.h Wed Apr 18 19:11:21 2012 (r234437) @@ -1,7 +1,7 @@ /* * $FreeBSD$ */ -#include <contrib/rdma/ib_verbs.h> +#include <rdma/ib_verbs.h> #include <netinet/in.h> /* @@ -92,6 +92,8 @@ struct krping_cb { int count; /* ping count */ int size; /* ping data size */ int validate; /* validate ping data */ + uint64_t memlimit; /* limit of the physical memory that + can be registered with dma_mr mode */ /* CM stuff */ struct rdma_cm_id *cm_id; /* connection on client side,*/ Modified: user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c ============================================================================== --- user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/contrib/rdma/krping/krping_dev.c Wed Apr 18 19:11:21 2012 (r234437) @@ -14,7 +14,6 @@ __FBSDID("$FreeBSD$"); #include <sys/types.h> -#include <sys/module.h> #include <sys/systm.h> /* uprintf */ #include <sys/errno.h> #include <sys/param.h> /* defines used in kernel.h */ @@ -51,6 +50,9 @@ typedef struct s_krping { /* vars */ static struct cdev *krping_dev; +#undef MODULE_VERSION +#include <sys/module.h> + static int krping_loader(struct module *m, int what, void *arg) { @@ -175,6 +177,4 @@ krping_write(struct cdev *dev, struct ui return(err); } -MODULE_DEPEND(krping, rdma_core, 1, 1, 1); -MODULE_DEPEND(krping, rdma_cma, 1, 1, 1); DEV_MODULE(krping,krping_loader,NULL); Modified: user/np/toe_iwarp/sys/modules/rdma/krping/Makefile ============================================================================== --- user/np/toe_iwarp/sys/modules/rdma/krping/Makefile Wed Apr 18 19:11:03 2012 (r234436) +++ user/np/toe_iwarp/sys/modules/rdma/krping/Makefile Wed Apr 18 19:11:21 2012 (r234437) @@ -6,5 +6,7 @@ RDMA= ${.CURDIR}/../../../contrib/rdma/k KMOD= krping SRCS= krping.c krping_dev.c getopt.c SRCS+= bus_if.h device_if.h opt_sched.h pci_if.h pcib_if.h +SRCS+= vnode_if.h +CFLAGS+= -I${.CURDIR}/../../../ofed/include .include <bsd.kmod.mk>help
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201204181911.q3IJBLCU046712>