Date: Thu, 1 Sep 2016 20:43:01 +0000 (UTC) From: Navdeep Parhar <np@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r305240 - in head/sys: dev/cxgbe dev/cxgbe/cxgbei modules/cxgbe/cxgbei Message-ID: <201609012043.u81Kh1ZM051875@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: np Date: Thu Sep 1 20:43:01 2016 New Revision: 305240 URL: https://svnweb.freebsd.org/changeset/base/305240 Log: cxgbe/cxgbei: Retire all DDP related code from cxgbei and switch to routines available in t4_tom to manage the iSCSI DDP page pod region. This adds the ability to use multiple DDP page sizes to the iSCSI driver, among other improvements. Sponsored by: Chelsio Communications Deleted: head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h Modified: head/sys/dev/cxgbe/cxgbei/cxgbei.c head/sys/dev/cxgbe/cxgbei/cxgbei.h head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c head/sys/dev/cxgbe/offload.h head/sys/dev/cxgbe/t4_main.c head/sys/dev/cxgbe/t4_sge.c head/sys/modules/cxgbe/cxgbei/Makefile Modified: head/sys/dev/cxgbe/cxgbei/cxgbei.c ============================================================================== --- head/sys/dev/cxgbe/cxgbei/cxgbei.c Thu Sep 1 20:38:59 2016 (r305239) +++ head/sys/dev/cxgbe/cxgbei/cxgbei.c Thu Sep 1 20:43:01 2016 (r305240) @@ -90,7 +90,6 @@ __FBSDID("$FreeBSD$"); #include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */ #include "tom/t4_tom.h" #include "cxgbei.h" -#include "cxgbei_ulp2_ddp.h" static int worker_thread_count; static struct cxgbei_worker_thread_softc *cwt_softc; @@ -101,376 +100,6 @@ struct icl_pdu *icl_cxgbei_new_pdu(int); void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *); void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *); -/* - * Direct Data Placement - - * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted - * final destination host-memory buffers based on the Initiator Task Tag (ITT) - * in Data-In or Target Task Tag (TTT) in Data-Out PDUs. - * The host memory address is programmed into h/w in the format of pagepod - * entries. - * The location of the pagepod entry is encoded into ddp tag which is used as - * the base for ITT/TTT. - */ - -/* - * functions to program the pagepod in h/w - */ -static void inline -ppod_set(struct pagepod *ppod, - struct cxgbei_ulp2_pagepod_hdr *hdr, - struct cxgbei_ulp2_gather_list *gl, - unsigned int pidx) -{ - int i; - - memcpy(ppod, hdr, sizeof(*hdr)); - - for (i = 0; i < (PPOD_PAGES + 1); i++, pidx++) { - ppod->addr[i] = pidx < gl->nelem ? - cpu_to_be64(gl->dma_sg[pidx].phys_addr) : 0ULL; - } -} - -static void inline -ppod_clear(struct pagepod *ppod) -{ - memset(ppod, 0, sizeof(*ppod)); -} - -static inline void -ulp_mem_io_set_hdr(struct adapter *sc, int tid, struct ulp_mem_io *req, - unsigned int wr_len, unsigned int dlen, - unsigned int pm_addr) -{ - struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1); - - INIT_ULPTX_WR(req, wr_len, 0, 0); - req->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | - V_ULP_MEMIO_ORDER(is_t4(sc)) | - V_T5_ULP_MEMIO_IMM(is_t5(sc))); - req->dlen = htonl(V_ULP_MEMIO_DATA_LEN(dlen >> 5)); - req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16) - | V_FW_WR_FLOWID(tid)); - req->lock_addr = htonl(V_ULP_MEMIO_ADDR(pm_addr >> 5)); - - idata->cmd_more = htonl(V_ULPTX_CMD(ULP_TX_SC_IMM)); - idata->len = htonl(dlen); -} - -#define ULPMEM_IDATA_MAX_NPPODS 1 /* 256/PPOD_SIZE */ -#define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */ - -static int -ppod_write_idata(struct cxgbei_data *ci, - struct cxgbei_ulp2_pagepod_hdr *hdr, - unsigned int idx, unsigned int npods, - struct cxgbei_ulp2_gather_list *gl, - unsigned int gl_pidx, struct toepcb *toep) -{ - u_int dlen = PPOD_SIZE * npods; - u_int pm_addr = idx * PPOD_SIZE + ci->llimit; - u_int wr_len = roundup(sizeof(struct ulp_mem_io) + - sizeof(struct ulptx_idata) + dlen, 16); - struct ulp_mem_io *req; - struct ulptx_idata *idata; - struct pagepod *ppod; - u_int i; - struct wrqe *wr; - struct adapter *sc = toep->vi->pi->adapter; - - wr = alloc_wrqe(wr_len, toep->ctrlq); - if (wr == NULL) { - CXGBE_UNIMPLEMENTED("ppod_write_idata: alloc_wrqe failure"); - return (ENOMEM); - } - - req = wrtod(wr); - memset(req, 0, wr_len); - ulp_mem_io_set_hdr(sc, toep->tid, req, wr_len, dlen, pm_addr); - idata = (struct ulptx_idata *)(req + 1); - - ppod = (struct pagepod *)(idata + 1); - for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES) { - if (!hdr) /* clear the pagepod */ - ppod_clear(ppod); - else /* set the pagepod */ - ppod_set(ppod, hdr, gl, gl_pidx); - } - - t4_wrq_tx(sc, wr); - return 0; -} - -int -t4_ddp_set_map(struct cxgbei_data *ci, void *iccp, - struct cxgbei_ulp2_pagepod_hdr *hdr, u_int idx, u_int npods, - struct cxgbei_ulp2_gather_list *gl, int reply) -{ - struct icl_cxgbei_conn *icc = (struct icl_cxgbei_conn *)iccp; - struct toepcb *toep = icc->toep; - int err; - unsigned int pidx = 0, w_npods = 0, cnt; - - /* - * on T4, if we use a mix of IMMD and DSGL with ULP_MEM_WRITE, - * the order would not be guaranteed, so we will stick with IMMD - */ - gl->tid = toep->tid; - gl->port_id = toep->vi->pi->port_id; - gl->egress_dev = (void *)toep->vi->ifp; - - /* send via immediate data */ - for (; w_npods < npods; idx += cnt, w_npods += cnt, - pidx += PPOD_PAGES) { - cnt = npods - w_npods; - if (cnt > ULPMEM_IDATA_MAX_NPPODS) - cnt = ULPMEM_IDATA_MAX_NPPODS; - err = ppod_write_idata(ci, hdr, idx, cnt, gl, pidx, toep); - if (err) { - printf("%s: ppod_write_idata failed\n", __func__); - break; - } - } - return err; -} - -void -t4_ddp_clear_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl, - u_int tag, u_int idx, u_int npods, struct icl_cxgbei_conn *icc) -{ - struct toepcb *toep = icc->toep; - int err = -1; - u_int pidx = 0; - u_int w_npods = 0; - u_int cnt; - - for (; w_npods < npods; idx += cnt, w_npods += cnt, - pidx += PPOD_PAGES) { - cnt = npods - w_npods; - if (cnt > ULPMEM_IDATA_MAX_NPPODS) - cnt = ULPMEM_IDATA_MAX_NPPODS; - err = ppod_write_idata(ci, NULL, idx, cnt, gl, 0, toep); - if (err) - break; - } -} - -static int -cxgbei_map_sg(struct cxgbei_sgl *sgl, struct ccb_scsiio *csio) -{ - unsigned int data_len = csio->dxfer_len; - unsigned int sgoffset = (uint64_t)csio->data_ptr & PAGE_MASK; - unsigned int nsge; - unsigned char *sgaddr = csio->data_ptr; - unsigned int len = 0; - - nsge = (csio->dxfer_len + sgoffset + PAGE_SIZE - 1) >> PAGE_SHIFT; - sgl->sg_addr = sgaddr; - sgl->sg_offset = sgoffset; - if (data_len < (PAGE_SIZE - sgoffset)) - len = data_len; - else - len = PAGE_SIZE - sgoffset; - - sgl->sg_length = len; - - data_len -= len; - sgaddr += len; - sgl = sgl+1; - - while (data_len > 0) { - sgl->sg_addr = sgaddr; - len = (data_len < PAGE_SIZE)? data_len: PAGE_SIZE; - sgl->sg_length = len; - sgaddr += len; - data_len -= len; - sgl = sgl + 1; - } - - return nsge; -} - -static int -cxgbei_map_sg_tgt(struct cxgbei_sgl *sgl, union ctl_io *io) -{ - unsigned int data_len, sgoffset, nsge; - unsigned char *sgaddr; - unsigned int len = 0, index = 0, ctl_sg_count, i; - struct ctl_sg_entry ctl_sg_entry, *ctl_sglist; - - if (io->scsiio.kern_sg_entries > 0) { - ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr; - ctl_sg_count = io->scsiio.kern_sg_entries; - } else { - ctl_sglist = &ctl_sg_entry; - ctl_sglist->addr = io->scsiio.kern_data_ptr; - ctl_sglist->len = io->scsiio.kern_data_len; - ctl_sg_count = 1; - } - - sgaddr = sgl->sg_addr = ctl_sglist[index].addr; - sgoffset = sgl->sg_offset = (uint64_t)sgl->sg_addr & PAGE_MASK; - data_len = ctl_sglist[index].len; - - if (data_len < (PAGE_SIZE - sgoffset)) - len = data_len; - else - len = PAGE_SIZE - sgoffset; - - sgl->sg_length = len; - - data_len -= len; - sgaddr += len; - sgl = sgl+1; - - len = 0; - for (i = 0; i< ctl_sg_count; i++) - len += ctl_sglist[i].len; - nsge = (len + sgoffset + PAGE_SIZE -1) >> PAGE_SHIFT; - while (data_len > 0) { - sgl->sg_addr = sgaddr; - len = (data_len < PAGE_SIZE)? data_len: PAGE_SIZE; - sgl->sg_length = len; - sgaddr += len; - data_len -= len; - sgl = sgl + 1; - if (data_len == 0) { - if (index == ctl_sg_count - 1) - break; - index++; - sgaddr = ctl_sglist[index].addr; - data_len = ctl_sglist[index].len; - } - } - - return nsge; -} - -static int -t4_sk_ddp_tag_reserve(struct cxgbei_data *ci, struct icl_cxgbei_conn *icc, - u_int xferlen, struct cxgbei_sgl *sgl, u_int sgcnt, u_int *ddp_tag) -{ - struct cxgbei_ulp2_gather_list *gl; - int err = -EINVAL; - struct toepcb *toep = icc->toep; - - gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt, ci, 0); - if (gl) { - err = cxgbei_ulp2_ddp_tag_reserve(ci, icc, toep->tid, - &ci->tag_format, ddp_tag, gl, 0, 0); - if (err) { - cxgbei_ulp2_ddp_release_gl(ci, gl); - } - } - - return err; -} - -static unsigned int -cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv, - struct ccb_scsiio *scmd, unsigned int *itt) -{ - struct icl_cxgbei_conn *icc = ic_to_icc(ic); - int xferlen = scmd->dxfer_len; - struct cxgbei_task_data *tdata = NULL; - struct cxgbei_sgl *sge = NULL; - struct toepcb *toep = icc->toep; - struct adapter *sc = td_adapter(toep->td); - struct cxgbei_data *ci = sc->iscsi_ulp_softc; - int err = -1; - - MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); - - tdata = (struct cxgbei_task_data *)*prv; - if (xferlen == 0 || tdata == NULL) - goto out; - if (xferlen < DDP_THRESHOLD) - goto out; - - if ((scmd->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) { - tdata->nsge = cxgbei_map_sg(tdata->sgl, scmd); - if (tdata->nsge == 0) { - CTR1(KTR_CXGBE, "%s: map_sg failed", __func__); - return 0; - } - sge = tdata->sgl; - - tdata->sc_ddp_tag = *itt; - - CTR3(KTR_CXGBE, "%s: *itt:0x%x sc_ddp_tag:0x%x", - __func__, *itt, tdata->sc_ddp_tag); - if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, - tdata->sc_ddp_tag)) { - err = t4_sk_ddp_tag_reserve(ci, icc, scmd->dxfer_len, - sge, tdata->nsge, &tdata->sc_ddp_tag); - } else { - CTR3(KTR_CXGBE, - "%s: itt:0x%x sc_ddp_tag:0x%x not usable", - __func__, *itt, tdata->sc_ddp_tag); - } - } -out: - if (err < 0) - tdata->sc_ddp_tag = - cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *itt); - - return tdata->sc_ddp_tag; -} - -static unsigned int -cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io, - unsigned int *ttt) -{ - struct icl_cxgbei_conn *icc = ic_to_icc(ic); - struct toepcb *toep = icc->toep; - struct adapter *sc = td_adapter(toep->td); - struct cxgbei_data *ci = sc->iscsi_ulp_softc; - struct cxgbei_task_data *tdata = NULL; - int xferlen, err = -1; - struct cxgbei_sgl *sge = NULL; - - MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); - - xferlen = (io->scsiio.kern_data_len - io->scsiio.ext_data_filled); - tdata = (struct cxgbei_task_data *)*prv; - if ((xferlen == 0) || (tdata == NULL)) - goto out; - if (xferlen < DDP_THRESHOLD) - goto out; - tdata->nsge = cxgbei_map_sg_tgt(tdata->sgl, io); - if (tdata->nsge == 0) { - CTR1(KTR_CXGBE, "%s: map_sg failed", __func__); - return 0; - } - sge = tdata->sgl; - - tdata->sc_ddp_tag = *ttt; - if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) { - err = t4_sk_ddp_tag_reserve(ci, icc, xferlen, sge, - tdata->nsge, &tdata->sc_ddp_tag); - } else { - CTR2(KTR_CXGBE, "%s: sc_ddp_tag:0x%x not usable", - __func__, tdata->sc_ddp_tag); - } -out: - if (err < 0) - tdata->sc_ddp_tag = - cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *ttt); - return tdata->sc_ddp_tag; -} - -static int -t4_sk_ddp_tag_release(struct icl_cxgbei_conn *icc, unsigned int ddp_tag) -{ - struct toepcb *toep = icc->toep; - struct adapter *sc = td_adapter(toep->td); - struct cxgbei_data *ci = sc->iscsi_ulp_softc; - - cxgbei_ulp2_ddp_tag_release(ci, ddp_tag, icc); - - return (0); -} - static void read_pdu_limits(struct adapter *sc, uint32_t *max_tx_pdu_len, uint32_t *max_rx_pdu_len) @@ -504,58 +133,43 @@ read_pdu_limits(struct adapter *sc, uint static int cxgbei_init(struct adapter *sc, struct cxgbei_data *ci) { - int nppods, bits, rc; - static const u_int pgsz_order[] = {0, 1, 2, 3}; + struct ppod_region *pr; + uint32_t r; + int rc; MPASS(sc->vres.iscsi.size > 0); + MPASS(ci != NULL); - ci->llimit = sc->vres.iscsi.start; - ci->ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1; read_pdu_limits(sc, &ci->max_tx_pdu_len, &ci->max_rx_pdu_len); - nppods = sc->vres.iscsi.size >> IPPOD_SIZE_SHIFT; - if (nppods <= 1024) - return (ENXIO); - - bits = fls(nppods); - if (bits > IPPOD_IDX_MAX_SIZE) - bits = IPPOD_IDX_MAX_SIZE; - nppods = (1 << (bits - 1)) - 1; - - rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR, - BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8, BUS_SPACE_MAXSIZE, - BUS_DMA_ALLOCNOW, NULL, NULL, &ci->ulp_ddp_tag); + ci->ddp_threshold = 2048; + pr = &ci->pr; + + r = t4_read_reg(sc, A_ULP_RX_ISCSI_PSZ); + rc = t4_init_ppod_region(pr, &sc->vres.iscsi, r, "iSCSI page pods"); if (rc != 0) { - device_printf(sc->dev, "%s: failed to create DMA tag: %u.\n", + device_printf(sc->dev, + "%s: failed to initialize the iSCSI page pod region: %u.\n", __func__, rc); return (rc); } - ci->colors = malloc(nppods * sizeof(char), M_CXGBE, M_NOWAIT | M_ZERO); - ci->gl_map = malloc(nppods * sizeof(struct cxgbei_ulp2_gather_list *), - M_CXGBE, M_NOWAIT | M_ZERO); - if (ci->colors == NULL || ci->gl_map == NULL) { - bus_dma_tag_destroy(ci->ulp_ddp_tag); - free(ci->colors, M_CXGBE); - free(ci->gl_map, M_CXGBE); - return (ENOMEM); + r = t4_read_reg(sc, A_ULP_RX_ISCSI_TAGMASK); + r &= V_ISCSITAGMASK(M_ISCSITAGMASK); + if (r != pr->pr_tag_mask) { + /* + * Recent firmwares are supposed to set up the the iSCSI tagmask + * but we'll do it ourselves it the computed value doesn't match + * what's in the register. + */ + device_printf(sc->dev, + "tagmask 0x%08x does not match computed mask 0x%08x.\n", r, + pr->pr_tag_mask); + t4_set_reg_field(sc, A_ULP_RX_ISCSI_TAGMASK, + V_ISCSITAGMASK(M_ISCSITAGMASK), pr->pr_tag_mask); } - mtx_init(&ci->map_lock, "ddp lock", NULL, MTX_DEF | MTX_DUPOK); - ci->nppods = nppods; - ci->idx_last = nppods; - ci->idx_bits = bits; - ci->idx_mask = (1 << bits) - 1; - ci->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1; - - ci->tag_format.sw_bits = bits; - ci->tag_format.rsvd_bits = bits; - ci->tag_format.rsvd_shift = IPPOD_IDX_SHIFT; - ci->tag_format.rsvd_mask = ci->idx_mask; - - t4_iscsi_init(sc, ci->idx_mask << IPPOD_IDX_SHIFT, pgsz_order); - - return (rc); + return (0); } static int @@ -772,47 +386,6 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const return (0); } -/* initiator */ -void -cxgbei_conn_task_reserve_itt(void *conn, void **prv, - void *scmd, unsigned int *itt) -{ - unsigned int tag; - tag = cxgbei_task_reserve_itt(conn, prv, scmd, itt); - if (tag) - *itt = htonl(tag); - return; -} - -/* target */ -void -cxgbei_conn_transfer_reserve_ttt(void *conn, void **prv, - void *scmd, unsigned int *ttt) -{ - unsigned int tag; - tag = cxgbei_task_reserve_ttt(conn, prv, scmd, ttt); - if (tag) - *ttt = htonl(tag); - return; -} - -void -cxgbei_cleanup_task(void *conn, void *ofld_priv) -{ - struct icl_conn *ic = (struct icl_conn *)conn; - struct icl_cxgbei_conn *icc = ic_to_icc(ic); - struct cxgbei_task_data *tdata = ofld_priv; - struct adapter *sc = icc->sc; - struct cxgbei_data *ci = sc->iscsi_ulp_softc; - - MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE); - MPASS(tdata != NULL); - - if (cxgbei_ulp2_is_ddp_tag(&ci->tag_format, tdata->sc_ddp_tag)) - t4_sk_ddp_tag_release(icc, tdata->sc_ddp_tag); - memset(tdata, 0, sizeof(*tdata)); -} - static int cxgbei_activate(struct adapter *sc) { @@ -834,7 +407,7 @@ cxgbei_activate(struct adapter *sc) } /* per-adapter softc for iSCSI */ - ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_NOWAIT); + ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_WAITOK); if (ci == NULL) return (ENOMEM); @@ -852,12 +425,13 @@ cxgbei_activate(struct adapter *sc) static int cxgbei_deactivate(struct adapter *sc) { + struct cxgbei_data *ci = sc->iscsi_ulp_softc; ASSERT_SYNCHRONIZED_OP(sc); - if (sc->iscsi_ulp_softc != NULL) { - cxgbei_ddp_cleanup(sc->iscsi_ulp_softc); - free(sc->iscsi_ulp_softc, M_CXGBE); + if (ci != NULL) { + t4_free_ppod_region(&ci->pr); + free(ci, M_CXGBE); sc->iscsi_ulp_softc = NULL; } Modified: head/sys/dev/cxgbe/cxgbei/cxgbei.h ============================================================================== --- head/sys/dev/cxgbe/cxgbei/cxgbei.h Thu Sep 1 20:38:59 2016 (r305239) +++ head/sys/dev/cxgbe/cxgbei/cxgbei.h Thu Sep 1 20:43:01 2016 (r305240) @@ -105,69 +105,18 @@ ip_to_icp(struct icl_pdu *ip) return (__containerof(ip, struct icl_cxgbei_pdu, ip)); } -struct cxgbei_sgl { - int sg_flag; - void *sg_addr; - void *sg_dma_addr; - size_t sg_offset; - size_t sg_length; -}; - -#define cxgbei_scsi_for_each_sg(_sgl, _sgel, _n, _i) \ - for (_i = 0, _sgel = (cxgbei_sgl*) (_sgl); _i < _n; _i++, \ - _sgel++) -#define sg_dma_addr(_sgel) _sgel->sg_dma_addr -#define sg_virt(_sgel) _sgel->sg_addr -#define sg_len(_sgel) _sgel->sg_length -#define sg_off(_sgel) _sgel->sg_offset -#define sg_next(_sgel) _sgel + 1 - -/* private data for each scsi task */ -struct cxgbei_task_data { - struct cxgbei_sgl sgl[256]; - u_int nsge; - u_int sc_ddp_tag; -}; - -struct cxgbei_ulp2_tag_format { - u_char sw_bits; - u_char rsvd_bits; - u_char rsvd_shift; - u_char filler[1]; - uint32_t rsvd_mask; -}; - struct cxgbei_data { - u_int llimit; - u_int ulimit; - u_int nppods; - u_int idx_last; - u_char idx_bits; - uint32_t idx_mask; - uint32_t rsvd_tag_mask; u_int max_tx_pdu_len; u_int max_rx_pdu_len; - struct mtx map_lock; - bus_dma_tag_t ulp_ddp_tag; - unsigned char *colors; - struct cxgbei_ulp2_gather_list **gl_map; + u_int ddp_threshold; + struct ppod_region pr; - struct cxgbei_ulp2_tag_format tag_format; }; -void cxgbei_conn_task_reserve_itt(void *, void **, void *, unsigned int *); -void cxgbei_conn_transfer_reserve_ttt(void *, void **, void *, unsigned int *); -void cxgbei_cleanup_task(void *, void *); +/* cxgbei.c */ u_int cxgbei_select_worker_thread(struct icl_cxgbei_conn *); -struct cxgbei_ulp2_pagepod_hdr; -int t4_ddp_set_map(struct cxgbei_data *, void *, - struct cxgbei_ulp2_pagepod_hdr *, u_int, u_int, - struct cxgbei_ulp2_gather_list *, int); -void t4_ddp_clear_map(struct cxgbei_data *, struct cxgbei_ulp2_gather_list *, - u_int, u_int, u_int, struct icl_cxgbei_conn *); - /* icl_cxgbei.c */ int icl_cxgbei_mod_load(void); int icl_cxgbei_mod_unload(void); Modified: head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c ============================================================================== --- head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c Thu Sep 1 20:38:59 2016 (r305239) +++ head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c Thu Sep 1 20:43:01 2016 (r305240) @@ -60,6 +60,8 @@ __FBSDID("$FreeBSD$"); #include <sys/uio.h> #include <machine/bus.h> #include <vm/uma.h> +#include <vm/vm.h> +#include <vm/pmap.h> #include <netinet/in.h> #include <netinet/in_pcb.h> #include <netinet/tcp.h> @@ -70,6 +72,28 @@ __FBSDID("$FreeBSD$"); #include <dev/iscsi/iscsi_proto.h> #include <icl_conn_if.h> +#include <cam/scsi/scsi_all.h> +#include <cam/scsi/scsi_da.h> +#include <cam/ctl/ctl_io.h> +#include <cam/ctl/ctl.h> +#include <cam/ctl/ctl_backend.h> +#include <cam/ctl/ctl_error.h> +#include <cam/ctl/ctl_frontend.h> +#include <cam/ctl/ctl_debug.h> +#include <cam/ctl/ctl_ha.h> +#include <cam/ctl/ctl_ioctl.h> + +#include <cam/cam.h> +#include <cam/cam_ccb.h> +#include <cam/cam_xpt.h> +#include <cam/cam_debug.h> +#include <cam/cam_sim.h> +#include <cam/cam_xpt_sim.h> +#include <cam/cam_xpt_periph.h> +#include <cam/cam_periph.h> +#include <cam/cam_compat.h> +#include <cam/scsi/scsi_message.h> + #include "common/common.h" #include "common/t4_tcb.h" #include "tom/t4_tom.h" @@ -90,8 +114,7 @@ static int recvspace = 1048576; SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN, &recvspace, 0, "Default receive socket buffer size"); -static uma_zone_t icl_transfer_zone; - +static uma_zone_t prsv_zone; static volatile u_int icl_cxgbei_ncons; #define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock) @@ -242,12 +265,6 @@ icl_cxgbei_conn_pdu_data_segment_length( return (icl_pdu_data_segment_length(request)); } -static uint32_t -icl_conn_build_tasktag(struct icl_conn *ic, uint32_t tag) -{ - return tag; -} - static struct mbuf * finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp) { @@ -776,55 +793,215 @@ icl_cxgbei_conn_close(struct icl_conn *i int icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip, - struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp) + struct ccb_scsiio *csio, uint32_t *ittp, void **arg) { - void *prv; + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct toepcb *toep = icc->toep; + struct adapter *sc = icc->sc; + struct cxgbei_data *ci = sc->iscsi_ulp_softc; + struct ppod_region *pr = &ci->pr; + struct ppod_reservation *prsv; + uint32_t itt; + int rc = 0; + + /* This is for the offload driver's state. Must not be set already. */ + MPASS(arg != NULL); + MPASS(*arg == NULL); + + if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN || + csio->dxfer_len < ci->ddp_threshold) { +no_ddp: + /* + * No DDP for this I/O. Allocate an ITT (based on the one + * passed in) that cannot be a valid hardware DDP tag in the + * iSCSI region. + */ + itt = *ittp & M_PPOD_TAG; + itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit; + *ittp = htobe32(itt); + MPASS(*arg == NULL); /* State is maintained for DDP only. */ + return (0); + } - *task_tagp = icl_conn_build_tasktag(ic, *task_tagp); + /* + * Reserve resources for DDP, update the itt that should be used in the + * PDU, and save DDP specific state for this I/O in *arg. + */ - prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO); - if (prv == NULL) - return (ENOMEM); + prsv = uma_zalloc(prsv_zone, M_NOWAIT); + if (prsv == NULL) { + rc = ENOMEM; + goto no_ddp; + } - *prvp = prv; + /* XXX add support for all CAM_DATA_ types */ + MPASS((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR); + rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr, + csio->dxfer_len, prsv); + if (rc != 0) { + uma_zfree(prsv_zone, prsv); + goto no_ddp; + } - cxgbei_conn_task_reserve_itt(ic, prvp, csio, task_tagp); + rc = t4_write_page_pods_for_buf(sc, toep->ofld_txq, toep->tid, prsv, + (vm_offset_t)csio->data_ptr, csio->dxfer_len); + if (rc != 0) { + t4_free_page_pods(prsv); + uma_zfree(prsv_zone, prsv); + goto no_ddp; + } + *ittp = htobe32(prsv->prsv_tag); + *arg = prsv; return (0); } void -icl_cxgbei_conn_task_done(struct icl_conn *ic, void *prv) +icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg) { - cxgbei_cleanup_task(ic, prv); - uma_zfree(icl_transfer_zone, prv); + if (arg != NULL) { + struct ppod_reservation *prsv = arg; + + t4_free_page_pods(prsv); + uma_zfree(prsv_zone, prsv); + } } +/* XXXNP: PDU should be passed in as parameter, like on the initiator. */ +#define io_to_request_pdu(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr) +#define io_to_ppod_reservation(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr) + int icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io, - uint32_t *transfer_tag, void **prvp) + uint32_t *tttp, void **arg) { - void *prv; + struct icl_cxgbei_conn *icc = ic_to_icc(ic); + struct toepcb *toep = icc->toep; + struct ctl_scsiio *ctsio = &io->scsiio; + struct adapter *sc = icc->sc; + struct cxgbei_data *ci = sc->iscsi_ulp_softc; + struct ppod_region *pr = &ci->pr; + struct ppod_reservation *prsv; + uint32_t ttt; + int xferlen, rc = 0, alias; + + /* This is for the offload driver's state. Must not be set already. */ + MPASS(arg != NULL); + MPASS(*arg == NULL); + + if (ctsio->ext_data_filled == 0) { + int first_burst; + struct icl_pdu *ip = io_to_request_pdu(io); + vm_offset_t buf; +#ifdef INVARIANTS + struct icl_cxgbei_pdu *icp = ip_to_icp(ip); + + MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE); + MPASS(ic == ip->ip_conn); + MPASS(ip->ip_bhs_mbuf != NULL); +#endif + first_burst = icl_pdu_data_segment_length(ip); + + /* + * Note that ICL calls conn_transfer_setup even if the first + * burst had everything and there's nothing left to transfer. + */ + MPASS(ctsio->kern_data_len >= first_burst); + xferlen = ctsio->kern_data_len; + if (xferlen - first_burst < ci->ddp_threshold) { +no_ddp: + /* + * No DDP for this transfer. Allocate a TTT (based on + * the one passed in) that cannot be a valid hardware + * DDP tag in the iSCSI region. + */ + ttt = *tttp & M_PPOD_TAG; + ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit; + *tttp = htobe32(ttt); + MPASS(io_to_ppod_reservation(io) == NULL); + return (0); + } - *transfer_tag = icl_conn_build_tasktag(ic, *transfer_tag); + if (ctsio->kern_sg_entries == 0) + buf = (vm_offset_t)ctsio->kern_data_ptr; + else if (ctsio->kern_sg_entries == 1) { + struct ctl_sg_entry *sgl = (void *)ctsio->kern_data_ptr; + + MPASS(sgl->len == xferlen); + buf = (vm_offset_t)sgl->addr; + } else { + rc = EAGAIN; /* XXX implement */ + goto no_ddp; + } - prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO); - if (prv == NULL) - return (ENOMEM); - *prvp = prv; + /* + * Reserve resources for DDP, update the ttt that should be used + * in the PDU, and save DDP specific state for this I/O. + */ - cxgbei_conn_transfer_reserve_ttt(ic, prvp, io, transfer_tag); + MPASS(io_to_ppod_reservation(io) == NULL); + prsv = uma_zalloc(prsv_zone, M_NOWAIT); + if (prsv == NULL) { + rc = ENOMEM; + goto no_ddp; + } + + rc = t4_alloc_page_pods_for_buf(pr, buf, xferlen, prsv); + if (rc != 0) { + uma_zfree(prsv_zone, prsv); + goto no_ddp; + } + + rc = t4_write_page_pods_for_buf(sc, toep->ofld_txq, toep->tid, + prsv, buf, xferlen); + if (rc != 0) { + t4_free_page_pods(prsv); + uma_zfree(prsv_zone, prsv); + goto no_ddp; + } + + *tttp = htobe32(prsv->prsv_tag); + io_to_ppod_reservation(io) = prsv; + *arg = ctsio; + return (0); + } + + /* + * In the middle of an I/O. A non-NULL page pod reservation indicates + * that a DDP buffer is being used for the I/O. + */ + + prsv = io_to_ppod_reservation(ctsio); + if (prsv == NULL) + goto no_ddp; + + alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift; + alias++; + prsv->prsv_tag &= ~pr->pr_alias_mask; + prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask; + + *tttp = htobe32(prsv->prsv_tag); + *arg = ctsio; return (0); } void -icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *prv) +icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg) { - cxgbei_cleanup_task(ic, prv); - uma_zfree(icl_transfer_zone, prv); + struct ctl_scsiio *ctsio = arg; + + if (ctsio != NULL && ctsio->kern_data_len == ctsio->ext_data_filled) { + struct ppod_reservation *prsv; + + prsv = io_to_ppod_reservation(ctsio); + MPASS(prsv != NULL); + + t4_free_page_pods(prsv); + uma_zfree(prsv_zone, prsv); + } } static void @@ -882,9 +1059,12 @@ icl_cxgbei_mod_load(void) { int rc; - icl_transfer_zone = uma_zcreate("icl_transfer", - 16 * 1024, NULL, NULL, NULL, NULL, - UMA_ALIGN_PTR, 0); + /* + * Space to track pagepod reservations. + */ + prsv_zone = uma_zcreate("Pagepod reservations", + sizeof(struct ppod_reservation), NULL, NULL, NULL, NULL, + CACHE_LINE_SIZE, 0); refcount_init(&icl_cxgbei_ncons, 0); @@ -903,7 +1083,7 @@ icl_cxgbei_mod_unload(void) icl_unregister("cxgbei", false); - uma_zdestroy(icl_transfer_zone); + uma_zdestroy(prsv_zone); return (0); } Modified: head/sys/dev/cxgbe/offload.h ============================================================================== --- head/sys/dev/cxgbe/offload.h Thu Sep 1 20:38:59 2016 (r305239) +++ head/sys/dev/cxgbe/offload.h Thu Sep 1 20:43:01 2016 (r305240) @@ -155,7 +155,6 @@ int t4_register_uld(struct uld_info *); int t4_unregister_uld(struct uld_info *); int t4_activate_uld(struct adapter *, int); int t4_deactivate_uld(struct adapter *, int); -void t4_iscsi_init(struct adapter *, u_int, const u_int *); int uld_active(struct adapter *, int); #endif #endif Modified: head/sys/dev/cxgbe/t4_main.c ============================================================================== --- head/sys/dev/cxgbe/t4_main.c Thu Sep 1 20:38:59 2016 (r305239) +++ head/sys/dev/cxgbe/t4_main.c Thu Sep 1 20:43:01 2016 (r305240) @@ -8929,16 +8929,6 @@ t4_db_dropped(struct adapter *sc) } #ifdef TCP_OFFLOAD -void -t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order) -{ - - t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask); - t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) | - V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) | - V_HPZ3(pgsz_order[3])); -} - static int toe_capability(struct vi_info *vi, int enable) { Modified: head/sys/dev/cxgbe/t4_sge.c ============================================================================== --- head/sys/dev/cxgbe/t4_sge.c Thu Sep 1 20:38:59 2016 (r305239) +++ head/sys/dev/cxgbe/t4_sge.c Thu Sep 1 20:43:01 2016 (r305240) @@ -575,11 +575,20 @@ t4_tweak_chip_settings(struct adapter *s V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])); t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v); - /* 4K, 16K, 64K, 256K DDP "page sizes" */ + /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */ v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6); t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v); - m = v = F_TDDPTAGTCB; + /* + * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been + * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we + * may have to deal with is MAXPHYS + 1 page. + */ + v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4); + t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v); + + /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */ + m = v = F_TDDPTAGTCB | F_ISCSITAGTCB; t4_set_reg_field(sc, A_ULP_RX_CTL, m, v); m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET | *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201609012043.u81Kh1ZM051875>