Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 24 May 2017 18:16:20 +0000 (UTC)
From:      Navdeep Parhar <np@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org
Subject:   svn commit: r318799 - in stable/10: contrib/ofed/libcxgb4/src sys/dev/cxgb/ulp/iw_cxgb sys/dev/cxgbe/iw_cxgbe
Message-ID:  <201705241816.v4OIGKBn048285@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: np
Date: Wed May 24 18:16:20 2017
New Revision: 318799
URL: https://svnweb.freebsd.org/changeset/base/318799

Log:
  MFC r311880, r314167, r316118, r316571, r316573, r316580, r316936-r316937,
  r316940, and r317410.
  
  r311880:
  The iw_cxgb and iw_cxgbe drivers should not use a FreeBSD device_t where
  a linuxkpi style device is expected.  If OFED/linuxkpi actually starts
  using this field then we'll have to figure out whether to create fake
  devices for these drivers or have linuxkpi deal with NULL device.
  
  This mismatch was first reported as part of D6585.
  
  r314167:
  cxgbe/iw_cxgbe: Minor changes for T6.
  
  r316118:
  cxgbe/iw_cxgbe: T6 has no limit on the amount of memory that can be
  registered in one ib_reg_phys_mr.
  
  r316571:
  cxgbe/iw_cxgbe: Remove bad cast that resulted in incorrect length for
  memory regions larger than 4GB.
  
  r316573:
  cxgbe/iw_cxgbe: Replace a magic constant with something more readable
  (and accurate).
  
  T4 and later have an extra bit for page shift so the maximum page size
  is 8TB (shift of 12 + 31) instead of 128MB (12 + 15).  This saves space
  in the chip's PBL (physical buffer list) when registering very large
  memory regions.
  
  r316580:
  cxgbe/iw_cxgbe: Remove another bad cast.  This should have been
  included in r316571.
  
  r316936:
  cxgbe/iw_cxgbe: hw supports 64K (not 32K) Protection Domains.
  
  r316937:
  cxgbe/iw_cxgbe: Report accurate page_size_cap in ib_query_device.
  
  r316940:
  cxgbe/iw_cxgbe: Report the actual values of various parameters as
  configured by the firmware.
  
  r317410:
  cxgbe/iw_cxgbe: Pull in some updates to c4iw_wait_for_reply from the
  iw_cxgb4 Linux driver.

Modified:
  stable/10/contrib/ofed/libcxgb4/src/t4.h
  stable/10/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c
  stable/10/sys/dev/cxgbe/iw_cxgbe/device.c
  stable/10/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h
  stable/10/sys/dev/cxgbe/iw_cxgbe/mem.c
  stable/10/sys/dev/cxgbe/iw_cxgbe/provider.c
  stable/10/sys/dev/cxgbe/iw_cxgbe/qp.c
  stable/10/sys/dev/cxgbe/iw_cxgbe/t4.h
Directory Properties:
  stable/10/   (props changed)

Modified: stable/10/contrib/ofed/libcxgb4/src/t4.h
==============================================================================
--- stable/10/contrib/ofed/libcxgb4/src/t4.h	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/contrib/ofed/libcxgb4/src/t4.h	Wed May 24 18:16:20 2017	(r318799)
@@ -102,7 +102,7 @@
 #define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 1)
 #define T4_MAX_NUM_STAG (1<<15)
 #define T4_MAX_MR_SIZE (~0ULL - 1)
-#define T4_PAGESIZE_MASK 0xffff000  /* 4KB-128MB */
+#define T4_PAGESIZE_MASK 0xffffffff000  /* 4KB-8TB */
 #define T4_STAG_UNSET 0xffffffff
 #define T4_FW_MAJ 0
 

Modified: stable/10/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c
==============================================================================
--- stable/10/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.c	Wed May 24 18:16:20 2017	(r318799)
@@ -1085,7 +1085,7 @@ int iwch_register_device(struct iwch_dev
 	memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
 	dev->ibdev.phys_port_cnt = sc->params.nports;
 	dev->ibdev.num_comp_vectors = 1;
-	dev->ibdev.dma_device = dev->rdev.adap->dev;
+	dev->ibdev.dma_device = NULL;
 	dev->ibdev.query_device = iwch_query_device;
 	dev->ibdev.query_port = iwch_query_port;
 	dev->ibdev.modify_port = iwch_modify_port;

Modified: stable/10/sys/dev/cxgbe/iw_cxgbe/device.c
==============================================================================
--- stable/10/sys/dev/cxgbe/iw_cxgbe/device.c	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgbe/iw_cxgbe/device.c	Wed May 24 18:16:20 2017	(r318799)
@@ -332,7 +332,7 @@ c4iw_modevent(module_t mod, int cmd, voi
 	case MOD_LOAD:
 		rc = c4iw_mod_load();
 		if (rc == 0)
-			printf("iw_cxgbe: Chelsio T4/T5 RDMA driver loaded.\n");
+			printf("iw_cxgbe: Chelsio T4/T5/T6 RDMA driver loaded.\n");
 		break;
 
 	case MOD_UNLOAD:

Modified: stable/10/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h
==============================================================================
--- stable/10/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h	Wed May 24 18:16:20 2017	(r318799)
@@ -157,49 +157,70 @@ static inline int c4iw_num_stags(struct 
 	return (int)(rdev->adap->vres.stag.size >> 5);
 }
 
-#define C4IW_WR_TO (10*HZ)
+#define C4IW_WR_TO (60*HZ)
 
 struct c4iw_wr_wait {
 	int ret;
-	atomic_t completion;
+	struct completion completion;
 };
 
 static inline void c4iw_init_wr_wait(struct c4iw_wr_wait *wr_waitp)
 {
 	wr_waitp->ret = 0;
-	atomic_set(&wr_waitp->completion, 0);
+	init_completion(&wr_waitp->completion);
 }
 
 static inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret)
 {
 	wr_waitp->ret = ret;
-	atomic_set(&wr_waitp->completion, 1);
-	wakeup(wr_waitp);
+	complete(&wr_waitp->completion);
 }
 
 static inline int
 c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp,
-    u32 hwtid, u32 qpid, const char *func)
+					u32 hwtid, u32 qpid, const char *func)
 {
 	struct adapter *sc = rdev->adap;
 	unsigned to = C4IW_WR_TO;
+	int ret;
+	int timedout = 0;
+	struct timeval t1, t2;
 
-	while (!atomic_read(&wr_waitp->completion)) {
-                tsleep(wr_waitp, 0, "c4iw_wait", to);
-                if (SIGPENDING(curthread)) {
-			printf("%s - Device %s not responding - "
-			    "tid %u qpid %u\n", func,
-			    device_get_nameunit(sc->dev), hwtid, qpid);
-                        if (c4iw_fatal_error(rdev)) {
-                                wr_waitp->ret = -EIO;
-                                break;
-                        }
-                        to = to << 2;
-                }
-        }
+	if (c4iw_fatal_error(rdev)) {
+		wr_waitp->ret = -EIO;
+		goto out;
+	}
+
+	getmicrotime(&t1);
+	do {
+		ret = wait_for_completion_timeout(&wr_waitp->completion, to);
+		if (!ret) {
+			getmicrotime(&t2);
+			timevalsub(&t2, &t1);
+			printf("%s - Device %s not responding after %ld.%06ld "
+			    "seconds - tid %u qpid %u\n", func,
+			    device_get_nameunit(sc->dev), t2.tv_sec, t2.tv_usec,
+			    hwtid, qpid);
+			if (c4iw_fatal_error(rdev)) {
+				wr_waitp->ret = -EIO;
+				break;
+			}
+			to = to << 2;
+			timedout = 1;
+		}
+	} while (!ret);
+
+out:
+	if (timedout) {
+		getmicrotime(&t2);
+		timevalsub(&t2, &t1);
+		printf("%s - Device %s reply after %ld.%06ld seconds - "
+		    "tid %u qpid %u\n", func, device_get_nameunit(sc->dev),
+		    t2.tv_sec, t2.tv_usec, hwtid, qpid);
+	}
 	if (wr_waitp->ret)
-		CTR4(KTR_IW_CXGBE, "%s: FW reply %d tid %u qpid %u",
-		    device_get_nameunit(sc->dev), wr_waitp->ret, hwtid, qpid);
+		CTR4(KTR_IW_CXGBE, "%p: FW reply %d tid %u qpid %u", sc,
+		    wr_waitp->ret, hwtid, qpid);
 	return (wr_waitp->ret);
 }
 

Modified: stable/10/sys/dev/cxgbe/iw_cxgbe/mem.c
==============================================================================
--- stable/10/sys/dev/cxgbe/iw_cxgbe/mem.c	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgbe/iw_cxgbe/mem.c	Wed May 24 18:16:20 2017	(r318799)
@@ -46,12 +46,15 @@ __FBSDID("$FreeBSD$");
 #define T4_ULPTX_MIN_IO 32
 #define C4IW_MAX_INLINE_SIZE 96
 
-static int mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
+static int
+mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
 {
-	return (is_t4(dev->rdev.adap) ||
+
+	return ((is_t4(dev->rdev.adap) ||
 		is_t5(dev->rdev.adap)) &&
-		length >= 8*1024*1024*1024ULL;
+		length >= 8*1024*1024*1024ULL);
 }
+
 static int
 write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
 {
@@ -342,7 +345,8 @@ static int build_phys_page_list(struct i
 	}
 
 	/* Find largest page shift we can use to cover buffers */
-	for (*shift = PAGE_SHIFT; *shift < 27; ++(*shift))
+	for (*shift = PAGE_SHIFT; *shift < PAGE_SHIFT + M_FW_RI_TPTE_PS;
+	    ++(*shift))
 		if ((1ULL << *shift) & mask)
 			break;
 
@@ -439,7 +443,7 @@ int c4iw_reregister_phys_mem(struct ib_m
 		mhp->attr.zbva = 0;
 		mhp->attr.va_fbo = *iova_start;
 		mhp->attr.page_size = shift - 12;
-		mhp->attr.len = (u32) total_size;
+		mhp->attr.len = total_size;
 		mhp->attr.pbl_size = npages;
 	}
 
@@ -511,7 +515,7 @@ struct ib_mr *c4iw_register_phys_mem(str
 	mhp->attr.va_fbo = *iova_start;
 	mhp->attr.page_size = shift - 12;
 
-	mhp->attr.len = (u32) total_size;
+	mhp->attr.len = total_size;
 	mhp->attr.pbl_size = npages;
 	ret = register_mem(rhp, php, mhp, shift);
 	if (ret)

Modified: stable/10/sys/dev/cxgbe/iw_cxgbe/provider.c
==============================================================================
--- stable/10/sys/dev/cxgbe/iw_cxgbe/provider.c	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgbe/iw_cxgbe/provider.c	Wed May 24 18:16:20 2017	(r318799)
@@ -188,7 +188,7 @@ static int c4iw_mmap(struct ib_ucontext 
 			    "%s:6 USER DB-GTS addr %p region %p, reglen %u",
 			    __func__, addr, va_udbs_res, len_udbs_res);
 #ifdef DOT5
-			if (is_t5(rdev->lldi.adapter_type) && map_udb_as_wc)
+			if (!is_t4(rdev->lldi.adapter_type) && map_udb_as_wc)
 				vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot);
 			else
 #endif
@@ -305,6 +305,7 @@ c4iw_query_device(struct ib_device *ibde
 {
 	struct c4iw_dev *dev = to_c4iw_dev(ibdev);
 	struct adapter *sc = dev->rdev.adap;
+	const int spg_ndesc = sc->params.sge.spg_len / EQ_ESIZE;
 
 	CTR3(KTR_IW_CXGBE, "%s ibdev %p, props %p", __func__, ibdev, props);
 
@@ -318,13 +319,15 @@ c4iw_query_device(struct ib_device *ibde
 	props->vendor_id = pci_get_vendor(sc->dev);
 	props->vendor_part_id = pci_get_device(sc->dev);
 	props->max_mr_size = T4_MAX_MR_SIZE;
-	props->max_qp = T4_MAX_NUM_QP;
-	props->max_qp_wr = T4_MAX_QP_DEPTH;
+	props->max_qp = sc->vres.qp.size / 2;
+	props->max_qp_wr = T4_MAX_QP_DEPTH(spg_ndesc);
 	props->max_sge = T4_MAX_RECV_SGE;
 	props->max_sge_rd = 1;
-	props->max_qp_rd_atom = c4iw_max_read_depth;
-	props->max_qp_init_rd_atom = c4iw_max_read_depth;
-	props->max_cq = T4_MAX_NUM_CQ;
+	props->max_res_rd_atom = sc->params.max_ird_adapter;
+	props->max_qp_rd_atom = min(sc->params.max_ordird_qp,
+	    c4iw_max_read_depth);
+	props->max_qp_init_rd_atom = props->max_qp_rd_atom;
+	props->max_cq = sc->vres.qp.size;
 	props->max_cqe = T4_MAX_CQ_DEPTH;
 	props->max_mr = c4iw_num_stags(&dev->rdev);
 	props->max_pd = T4_MAX_NUM_PD;
@@ -427,7 +430,7 @@ c4iw_register_device(struct c4iw_dev *de
 	strlcpy(ibdev->node_desc, C4IW_NODE_DESC, sizeof(ibdev->node_desc));
 	ibdev->phys_port_cnt = sc->params.nports;
 	ibdev->num_comp_vectors = 1;
-	ibdev->dma_device = sc->dev;
+	ibdev->dma_device = NULL;
 	ibdev->query_device = c4iw_query_device;
 	ibdev->query_port = c4iw_query_port;
 	ibdev->modify_port = c4iw_modify_port;

Modified: stable/10/sys/dev/cxgbe/iw_cxgbe/qp.c
==============================================================================
--- stable/10/sys/dev/cxgbe/iw_cxgbe/qp.c	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgbe/iw_cxgbe/qp.c	Wed May 24 18:16:20 2017	(r318799)
@@ -132,6 +132,7 @@ static int create_qp(struct c4iw_rdev *r
 	int ret;
 	int eqsize;
 	struct wrqe *wr;
+	const int spg_ndesc = sc->params.sge.spg_len / EQ_ESIZE;
 
 	wq->sq.qid = c4iw_get_qpid(rdev, uctx);
 	if (!wq->sq.qid)
@@ -213,8 +214,7 @@ static int create_qp(struct c4iw_rdev *r
 	res->u.sqrq.op = FW_RI_RES_OP_WRITE;
 
 	/* eqsize is the number of 64B entries plus the status page size. */
-	eqsize = wq->sq.size * T4_SQ_NUM_SLOTS +
-	    (sc->params.sge.spg_len / EQ_ESIZE);
+	eqsize = wq->sq.size * T4_SQ_NUM_SLOTS + spg_ndesc;
 
 	res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
 		V_FW_RI_RES_WR_HOSTFCMODE(0) |	/* no host cidx updates */
@@ -236,8 +236,7 @@ static int create_qp(struct c4iw_rdev *r
 	res->u.sqrq.op = FW_RI_RES_OP_WRITE;
 
 	/* eqsize is the number of 64B entries plus the status page size. */
-	eqsize = wq->rq.size * T4_RQ_NUM_SLOTS +
-	    (sc->params.sge.spg_len / EQ_ESIZE);
+	eqsize = wq->rq.size * T4_RQ_NUM_SLOTS + spg_ndesc;
 	res->u.sqrq.fetchszm_to_iqid = cpu_to_be32(
 		V_FW_RI_RES_WR_HOSTFCMODE(0) |	/* no host cidx updates */
 		V_FW_RI_RES_WR_CPRIO(0) |	/* don't keep in chip cache */
@@ -1522,7 +1521,7 @@ c4iw_create_qp(struct ib_pd *pd, struct 
 	struct c4iw_create_qp_resp uresp;
 	int sqsize, rqsize;
 	struct c4iw_ucontext *ucontext;
-	int ret;
+	int ret, spg_ndesc;
 	struct c4iw_mm_entry *mm1, *mm2, *mm3, *mm4;
 
 	CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd);
@@ -1540,12 +1539,13 @@ c4iw_create_qp(struct ib_pd *pd, struct 
 	if (attrs->cap.max_inline_data > T4_MAX_SEND_INLINE)
 		return ERR_PTR(-EINVAL);
 
+	spg_ndesc = rhp->rdev.adap->params.sge.spg_len / EQ_ESIZE;
 	rqsize = roundup(attrs->cap.max_recv_wr + 1, 16);
-	if (rqsize > T4_MAX_RQ_SIZE)
+	if (rqsize > T4_MAX_RQ_SIZE(spg_ndesc))
 		return ERR_PTR(-E2BIG);
 
 	sqsize = roundup(attrs->cap.max_send_wr + 1, 16);
-	if (sqsize > T4_MAX_SQ_SIZE)
+	if (sqsize > T4_MAX_SQ_SIZE(spg_ndesc))
 		return ERR_PTR(-E2BIG);
 
 	ucontext = pd->uobject ? to_c4iw_ucontext(pd->uobject->context) : NULL;
@@ -1555,9 +1555,10 @@ c4iw_create_qp(struct ib_pd *pd, struct 
 	if (!qhp)
 		return ERR_PTR(-ENOMEM);
 	qhp->wq.sq.size = sqsize;
-	qhp->wq.sq.memsize = (sqsize + 1) * sizeof *qhp->wq.sq.queue;
+	qhp->wq.sq.memsize = (sqsize + spg_ndesc) * sizeof *qhp->wq.sq.queue +
+	    16 * sizeof(__be64);
 	qhp->wq.rq.size = rqsize;
-	qhp->wq.rq.memsize = (rqsize + 1) * sizeof *qhp->wq.rq.queue;
+	qhp->wq.rq.memsize = (rqsize + spg_ndesc) * sizeof *qhp->wq.rq.queue;
 
 	if (ucontext) {
 		qhp->wq.sq.memsize = roundup(qhp->wq.sq.memsize, PAGE_SIZE);

Modified: stable/10/sys/dev/cxgbe/iw_cxgbe/t4.h
==============================================================================
--- stable/10/sys/dev/cxgbe/iw_cxgbe/t4.h	Wed May 24 18:14:57 2017	(r318798)
+++ stable/10/sys/dev/cxgbe/iw_cxgbe/t4.h	Wed May 24 18:16:20 2017	(r318799)
@@ -59,21 +59,17 @@
 #define  CIDXINC_SHIFT     0
 #define  CIDXINC(x)        ((x) << CIDXINC_SHIFT)
 
-#define T4_MAX_NUM_QP (1<<16)
-#define T4_MAX_NUM_CQ (1<<15)
-#define T4_MAX_NUM_PD (1<<15)
-#define T4_EQ_STATUS_ENTRIES (L1_CACHE_BYTES > 64 ? 2 : 1)
-#define T4_MAX_EQ_SIZE (65520 - T4_EQ_STATUS_ENTRIES)
-#define T4_MAX_IQ_SIZE (65520 - 1)
-#define T4_MAX_RQ_SIZE (8192 - T4_EQ_STATUS_ENTRIES)
-#define T4_MAX_SQ_SIZE (T4_MAX_EQ_SIZE - 1)
-#define T4_MAX_QP_DEPTH (T4_MAX_RQ_SIZE - 1)
-#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 1)
+#define T4_MAX_NUM_PD 65536
+#define T4_MAX_EQ_SIZE 65520
+#define T4_MAX_IQ_SIZE 65520
+#define T4_MAX_RQ_SIZE(n) (8192 - (n) - 1)
+#define T4_MAX_SQ_SIZE(n) (T4_MAX_EQ_SIZE - (n) - 1)
+#define T4_MAX_QP_DEPTH(n) (T4_MAX_RQ_SIZE(n))
+#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 2)
 #define T4_MAX_MR_SIZE (~0ULL - 1)
-#define T4_PAGESIZE_MASK 0xffff000  /* 4KB-128MB */
+#define T4_PAGESIZE_MASK 0xffffffff000  /* 4KB-8TB */
 #define T4_STAG_UNSET 0xffffffff
 #define T4_FW_MAJ 0
-#define T4_EQ_STATUS_ENTRIES (L1_CACHE_BYTES > 64 ? 2 : 1)
 #define A_PCIE_MA_SYNC 0x30b4
 
 struct t4_status_page {



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201705241816.v4OIGKBn048285>