Date: Tue, 19 Sep 2023 14:12:04 GMT From: Eric van Gyzen <vangyzen@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: a687910fc435 - main - Cleanup pthread locks in ofed RDMA verbs Message-ID: <202309191412.38JEC4go072915@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by vangyzen: URL: https://cgit.FreeBSD.org/src/commit/?id=a687910fc4352117413b8e0275383e4c687d4c4c commit a687910fc4352117413b8e0275383e4c687d4c4c Author: Sean Lim <sean.lim@dell.com> AuthorDate: 2023-09-18 23:56:30 +0000 Commit: Eric van Gyzen <vangyzen@FreeBSD.org> CommitDate: 2023-09-19 14:10:42 +0000 Cleanup pthread locks in ofed RDMA verbs On FreeBSD, pthread mutex, cond, and spinlocks allocate memory. On Linux-based systems, these calls do not allocate memory. So there was a safe assumption that the ofed RDMA verb calls do not need to explicitly destroy the pthread locks. This assumption is false on FreeBSD. So let us rearrange the code to cleanup the pthread locks. Reviewed by: delphij MFC after: 2 weeks Sponsored by: Dell EMC Isilon Differential Revision: https://reviews.freebsd.org/D41105 --- contrib/ofed/libcxgb4/dev.c | 9 ++- contrib/ofed/libcxgb4/verbs.c | 79 +++++++++++++---------- contrib/ofed/libibcm/cm.c | 3 +- contrib/ofed/libibverbs/cmd.c | 110 ++++++++++++++++++++++++++++----- contrib/ofed/libibverbs/device.c | 87 +++++++++++++++++++++++--- contrib/ofed/libibverbs/driver.h | 5 +- contrib/ofed/libibverbs/libibverbs.map | 3 + contrib/ofed/libibverbs/verbs.c | 42 ++++++++++--- contrib/ofed/libibverbs/verbs.h | 5 +- contrib/ofed/libmlx4/mlx4.c | 39 ++++++++++-- contrib/ofed/libmlx4/mlx4.h | 3 +- contrib/ofed/libmlx4/srq.c | 15 ++++- contrib/ofed/libmlx4/verbs.c | 28 +++++++-- contrib/ofed/libmlx5/mlx5.c | 68 +++++++++++++++++--- contrib/ofed/libmlx5/verbs.c | 63 +++++++++++++++---- contrib/ofed/librdmacm/cma.c | 3 +- 16 files changed, 453 insertions(+), 109 deletions(-) diff --git a/contrib/ofed/libcxgb4/dev.c b/contrib/ofed/libcxgb4/dev.c index 89494f9b46d7..d3c289dad9f2 100644 --- a/contrib/ofed/libcxgb4/dev.c +++ b/contrib/ofed/libcxgb4/dev.c @@ -519,7 +519,9 @@ found: return NULL; } - pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE); + if (pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE)) + goto err; + dev->ibv_dev.ops = &c4iw_dev_ops; dev->chip_version = CHELSIO_CHIP_VERSION(hca_table[i].device >> 8); dev->abi_version = abi_version; @@ -554,6 +556,11 @@ found: } return &dev->ibv_dev; + +err: + free(dev); + + return NULL; } static __attribute__((constructor)) void cxgb4_register_driver(void) diff --git a/contrib/ofed/libcxgb4/verbs.c b/contrib/ofed/libcxgb4/verbs.c index 04d765dff3f6..4e44b2285fff 100644 --- a/contrib/ofed/libcxgb4/verbs.c +++ b/contrib/ofed/libcxgb4/verbs.c @@ -190,7 +190,9 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, PDBG("%s c4iw_create_cq_resp reserved field modified by kernel\n", __FUNCTION__); - pthread_spin_init(&chp->lock, PTHREAD_PROCESS_PRIVATE); + ret = pthread_spin_init(&chp->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) + goto err2; #ifdef STALL_DETECTION gettimeofday(&chp->time, NULL); #endif @@ -203,12 +205,12 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, chp->cq.queue = mmap(NULL, chp->cq.memsize, PROT_READ|PROT_WRITE, MAP_SHARED, context->cmd_fd, resp.key); if (chp->cq.queue == MAP_FAILED) - goto err2; + goto err3; chp->cq.ugts = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, context->cmd_fd, resp.gts_key); if (chp->cq.ugts == MAP_FAILED) - goto err3; + goto err4; if (dev_is_t4(chp->rhp)) chp->cq.ugts += 1; @@ -216,7 +218,7 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, chp->cq.ugts += 5; chp->cq.sw_queue = calloc(chp->cq.size, sizeof *chp->cq.queue); if (!chp->cq.sw_queue) - goto err4; + goto err5; PDBG("%s cqid 0x%x key %" PRIx64 " va %p memsize %lu gts_key %" PRIx64 " va %p qid_mask 0x%x\n", @@ -228,10 +230,12 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, pthread_spin_unlock(&dev->lock); INC_STAT(cq); return &chp->ibv_cq; -err4: +err5: munmap(MASKED(chp->cq.ugts), c4iw_page_size); -err3: +err4: munmap(chp->cq.queue, chp->cq.memsize); +err3: + pthread_spin_destroy(&chp->lock); err2: (void)ibv_cmd_destroy_cq(&chp->ibv_cq); err1: @@ -265,6 +269,7 @@ int c4iw_destroy_cq(struct ibv_cq *ibcq) if (ret) { return ret; } + verbs_cleanup_cq(ibcq); munmap(MASKED(chp->cq.ugts), c4iw_page_size); munmap(chp->cq.queue, chp->cq.memsize); @@ -273,6 +278,7 @@ int c4iw_destroy_cq(struct ibv_cq *ibcq) pthread_spin_unlock(&dev->lock); free(chp->cq.sw_queue); + pthread_spin_destroy(&chp->lock); free(chp); return 0; } @@ -337,38 +343,40 @@ static struct ibv_qp *create_qp_v0(struct ibv_pd *pd, qhp->wq.rq.qid = resp.rqid; qhp->wq.rq.size = resp.rq_size; qhp->wq.rq.memsize = resp.rq_memsize; - pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE); + ret = pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) + goto err3; dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.sq_db_gts_key); if (dbva == MAP_FAILED) - goto err3; + goto err4; qhp->wq.sq.udb = dbva; qhp->wq.sq.queue = mmap(NULL, qhp->wq.sq.memsize, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.sq_key); if (qhp->wq.sq.queue == MAP_FAILED) - goto err4; + goto err5; dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.rq_db_gts_key); if (dbva == MAP_FAILED) - goto err5; + goto err6; qhp->wq.rq.udb = dbva; qhp->wq.rq.queue = mmap(NULL, qhp->wq.rq.memsize, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.rq_key); if (qhp->wq.rq.queue == MAP_FAILED) - goto err6; + goto err7; qhp->wq.sq.sw_sq = calloc(qhp->wq.sq.size, sizeof (struct t4_swsqe)); if (!qhp->wq.sq.sw_sq) - goto err7; + goto err8; qhp->wq.rq.sw_rq = calloc(qhp->wq.rq.size, sizeof (uint64_t)); if (!qhp->wq.rq.sw_rq) - goto err8; + goto err9; PDBG("%s sq dbva %p sq qva %p sq depth %u sq memsize %lu " " rq dbva %p rq qva %p rq depth %u rq memsize %lu\n", @@ -385,16 +393,18 @@ static struct ibv_qp *create_qp_v0(struct ibv_pd *pd, pthread_spin_unlock(&dev->lock); INC_STAT(qp); return &qhp->ibv_qp; -err8: +err9: free(qhp->wq.sq.sw_sq); -err7: +err8: munmap((void *)qhp->wq.rq.queue, qhp->wq.rq.memsize); -err6: +err7: munmap(MASKED(qhp->wq.rq.udb), c4iw_page_size); -err5: +err6: munmap((void *)qhp->wq.sq.queue, qhp->wq.sq.memsize); -err4: +err5: munmap(MASKED(qhp->wq.sq.udb), c4iw_page_size); +err4: + pthread_spin_destroy(&qhp->lock); err3: (void)ibv_cmd_destroy_qp(&qhp->ibv_qp); err2: @@ -448,12 +458,14 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, fprintf(stderr, "libcxgb4 warning - downlevel iw_cxgb4 driver. " "MA workaround disabled.\n"); } - pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE); + ret = pthread_spin_init(&qhp->lock, PTHREAD_PROCESS_PRIVATE); + if (ret) + goto err3; dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.sq_db_gts_key); if (dbva == MAP_FAILED) - goto err3; + goto err4; qhp->wq.sq.udb = dbva; if (!dev_is_t4(qhp->rhp)) { unsigned long segment_offset = 128 * (qhp->wq.sq.qid & @@ -471,12 +483,12 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, PROT_READ|PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.sq_key); if (qhp->wq.sq.queue == MAP_FAILED) - goto err4; + goto err5; dbva = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.rq_db_gts_key); if (dbva == MAP_FAILED) - goto err5; + goto err6; qhp->wq.rq.udb = dbva; if (!dev_is_t4(qhp->rhp)) { unsigned long segment_offset = 128 * (qhp->wq.rq.qid & @@ -493,22 +505,22 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, PROT_READ|PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.rq_key); if (qhp->wq.rq.queue == MAP_FAILED) - goto err6; + goto err7; qhp->wq.sq.sw_sq = calloc(qhp->wq.sq.size, sizeof (struct t4_swsqe)); if (!qhp->wq.sq.sw_sq) - goto err7; + goto err8; qhp->wq.rq.sw_rq = calloc(qhp->wq.rq.size, sizeof (uint64_t)); if (!qhp->wq.rq.sw_rq) - goto err8; + goto err9; if (t4_sq_onchip(&qhp->wq)) { qhp->wq.sq.ma_sync = mmap(NULL, c4iw_page_size, PROT_WRITE, MAP_SHARED, pd->context->cmd_fd, resp.ma_sync_key); if (qhp->wq.sq.ma_sync == MAP_FAILED) - goto err9; + goto err10; qhp->wq.sq.ma_sync += (A_PCIE_MA_SYNC & (c4iw_page_size - 1)); } @@ -534,18 +546,20 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, pthread_spin_unlock(&dev->lock); INC_STAT(qp); return &qhp->ibv_qp; -err9: +err10: free(qhp->wq.rq.sw_rq); -err8: +err9: free(qhp->wq.sq.sw_sq); -err7: +err8: munmap((void *)qhp->wq.rq.queue, qhp->wq.rq.memsize); -err6: +err7: munmap(MASKED(qhp->wq.rq.udb), c4iw_page_size); -err5: +err6: munmap((void *)qhp->wq.sq.queue, qhp->wq.sq.memsize); -err4: +err5: munmap(MASKED(qhp->wq.sq.udb), c4iw_page_size); +err4: + pthread_spin_destroy(&qhp->lock); err3: (void)ibv_cmd_destroy_qp(&qhp->ibv_qp); err2: @@ -625,6 +639,7 @@ int c4iw_destroy_qp(struct ibv_qp *ibqp) free(qhp->wq.rq.sw_rq); free(qhp->wq.sq.sw_sq); + pthread_spin_destroy(&qhp->lock); free(qhp); return 0; } diff --git a/contrib/ofed/libibcm/cm.c b/contrib/ofed/libibcm/cm.c index 07ba481afa3d..da3412eb2fce 100644 --- a/contrib/ofed/libibcm/cm.c +++ b/contrib/ofed/libibcm/cm.c @@ -232,7 +232,8 @@ static struct cm_id_private *ib_cm_alloc_id(struct ib_cm_device *device, memset(cm_id_priv, 0, sizeof *cm_id_priv); cm_id_priv->id.device = device; cm_id_priv->id.context = context; - pthread_mutex_init(&cm_id_priv->mut, NULL); + if (pthread_mutex_init(&cm_id_priv->mut, NULL)) + goto err; if (pthread_cond_init(&cm_id_priv->cond, NULL)) goto err; diff --git a/contrib/ofed/libibverbs/cmd.c b/contrib/ofed/libibverbs/cmd.c index 0a9cc3831d41..488ffedd146b 100644 --- a/contrib/ofed/libibverbs/cmd.c +++ b/contrib/ofed/libibverbs/cmd.c @@ -679,6 +679,7 @@ int ibv_cmd_create_srq_ex(struct ibv_context *context, struct ibv_create_srq_resp *resp, size_t resp_size) { struct verbs_xrcd *vxrcd = NULL; + int ret = 0; IBV_INIT_CMD_RESP(cmd, cmd_size, CREATE_XSRQ, resp, resp_size); @@ -705,8 +706,17 @@ int ibv_cmd_create_srq_ex(struct ibv_context *context, cmd->cq_handle = attr_ex->cq->handle; } - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; + ret = pthread_mutex_init(&srq->srq.mutex, NULL); + if (ret) + goto err; + ret = pthread_cond_init(&srq->srq.cond, NULL); + if (ret) + goto err_mutex; + + if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) { + ret = errno; + goto err_cond; + } (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); @@ -715,8 +725,6 @@ int ibv_cmd_create_srq_ex(struct ibv_context *context, srq->srq.srq_context = attr_ex->srq_context; srq->srq.pd = attr_ex->pd; srq->srq.events_completed = 0; - pthread_mutex_init(&srq->srq.mutex, NULL); - pthread_cond_init(&srq->srq.cond, NULL); /* * check that the last field is available. @@ -744,6 +752,12 @@ int ibv_cmd_create_srq_ex(struct ibv_context *context, attr_ex->attr.max_sge = resp->max_sge; return 0; +err_cond: + pthread_cond_destroy(&srq->srq.cond); +err_mutex: + pthread_mutex_destroy(&srq->srq.mutex); +err: + return ret; } @@ -837,6 +851,9 @@ int ibv_cmd_destroy_srq(struct ibv_srq *srq) pthread_cond_wait(&srq->cond, &srq->mutex); pthread_mutex_unlock(&srq->mutex); + pthread_cond_destroy(&srq->cond); + pthread_mutex_destroy(&srq->mutex); + return 0; } @@ -887,6 +904,31 @@ static int create_qp_ex_common(struct verbs_qp *qp, return 0; } +static int create_qp_handle_resp_common_cleanup(struct verbs_qp *qp) +{ + pthread_cond_destroy(&qp->qp.cond); + pthread_mutex_destroy(&qp->qp.mutex); +} + +static int create_qp_handle_resp_common_init(struct verbs_qp *qp) +{ + int ret = 0; + + ret = pthread_mutex_init(&qp->qp.mutex, NULL); + if (ret) + return ret; + ret = pthread_cond_init(&qp->qp.cond, NULL); + if (ret) + goto err; + + return ret; + +err: + pthread_mutex_destroy(&qp->qp.mutex); + + return ret; +} + static void create_qp_handle_resp_common(struct ibv_context *context, struct verbs_qp *qp, struct ibv_qp_init_attr_ex *qp_attr, @@ -913,8 +955,6 @@ static void create_qp_handle_resp_common(struct ibv_context *context, qp->qp.qp_type = qp_attr->qp_type; qp->qp.state = IBV_QPS_RESET; qp->qp.events_completed = 0; - pthread_mutex_init(&qp->qp.mutex, NULL); - pthread_cond_init(&qp->qp.cond, NULL); qp->comp_mask = 0; if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz) && @@ -977,9 +1017,15 @@ int ibv_cmd_create_qp_ex2(struct ibv_context *context, cmd->comp_mask = IBV_CREATE_QP_EX_KERNEL_MASK_IND_TABLE; } + err = create_qp_handle_resp_common_init(qp); + if (err) + return err; + err = write(context->cmd_fd, cmd, cmd_size); - if (err != cmd_size) - return errno; + if (err != cmd_size) { + err = errno; + goto err; + } (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); @@ -987,6 +1033,11 @@ int ibv_cmd_create_qp_ex2(struct ibv_context *context, vqp_sz); return 0; + +err: + create_qp_handle_resp_common_cleanup(qp); + + return err; } int ibv_cmd_create_qp_ex(struct ibv_context *context, @@ -1008,8 +1059,15 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context, if (err) return err; - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; + err = create_qp_handle_resp_common_init(qp); + if (err) + return err; + + err = write(context->cmd_fd, cmd, cmd_size); + if (err != cmd_size) { + err = errno; + goto err; + } (void)VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); @@ -1032,6 +1090,11 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context, create_qp_handle_resp_common(context, qp, attr_ex, resp, vxrcd, vqp_sz); return 0; + +err: + create_qp_handle_resp_common_cleanup(qp); + + return err; } int ibv_cmd_create_qp(struct ibv_pd *pd, @@ -1098,6 +1161,7 @@ int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, struct ibv_open_qp *cmd, size_t cmd_size, struct ibv_create_qp_resp *resp, size_t resp_size) { + int err = 0; struct verbs_xrcd *xrcd; IBV_INIT_CMD_RESP(cmd, cmd_size, OPEN_QP, resp, resp_size); @@ -1115,8 +1179,18 @@ int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, cmd->qpn = attr->qp_num; cmd->qp_type = attr->qp_type; - if (write(context->cmd_fd, cmd, cmd_size) != cmd_size) - return errno; + err = pthread_mutex_init(&qp->qp.mutex, NULL); + if (err) + return err; + err = pthread_cond_init(&qp->qp.cond, NULL); + if (err) + goto err_mutex; + + err = write(context->cmd_fd, cmd, cmd_size); + if (err != cmd_size) { + err = errno; + goto err_cond; + } (void) VALGRIND_MAKE_MEM_DEFINED(resp, resp_size); @@ -1131,8 +1205,6 @@ int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, qp->qp.qp_type = attr->qp_type; qp->qp.state = IBV_QPS_UNKNOWN; qp->qp.events_completed = 0; - pthread_mutex_init(&qp->qp.mutex, NULL); - pthread_cond_init(&qp->qp.cond, NULL); qp->comp_mask = 0; if (vext_field_avail(struct verbs_qp, xrcd, vqp_sz)) { qp->comp_mask = VERBS_QP_XRCD; @@ -1140,6 +1212,13 @@ int ibv_cmd_open_qp(struct ibv_context *context, struct verbs_qp *qp, } return 0; + +err_cond: + pthread_cond_destroy(&qp->qp.cond); +err_mutex: + pthread_mutex_destroy(&qp->qp.mutex); + + return err; } int ibv_cmd_query_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, @@ -1644,6 +1723,9 @@ int ibv_cmd_destroy_qp(struct ibv_qp *qp) pthread_cond_wait(&qp->cond, &qp->mutex); pthread_mutex_unlock(&qp->mutex); + pthread_cond_destroy(&qp->cond); + pthread_mutex_destroy(&qp->mutex); + return 0; } diff --git a/contrib/ofed/libibverbs/device.c b/contrib/ofed/libibverbs/device.c index d5cd2173cd8b..c3d0dbf573ab 100644 --- a/contrib/ofed/libibverbs/device.c +++ b/contrib/ofed/libibverbs/device.c @@ -132,13 +132,22 @@ __be64 __ibv_get_device_guid(struct ibv_device *device) } default_symver(__ibv_get_device_guid, ibv_get_device_guid); -void verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context, +int verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context, struct ibv_comp_channel *channel, void *cq_context) { + int err = 0; + cq->context = context; cq->channel = channel; + err = pthread_mutex_init(&cq->mutex, NULL); + if (err) + return err; + err = pthread_cond_init(&cq->cond, NULL); + if (err) + goto err; + if (cq->channel) { pthread_mutex_lock(&context->mutex); ++cq->channel->refcnt; @@ -148,8 +157,19 @@ void verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context, cq->cq_context = cq_context; cq->comp_events_completed = 0; cq->async_events_completed = 0; - pthread_mutex_init(&cq->mutex, NULL); - pthread_cond_init(&cq->cond, NULL); + + return err; + +err: + pthread_mutex_destroy(&cq->mutex); + + return err; +} + +void verbs_cleanup_cq(struct ibv_cq *cq) +{ + pthread_cond_destroy(&cq->cond); + pthread_mutex_destroy(&cq->mutex); } static struct ibv_cq_ex * @@ -158,6 +178,7 @@ __lib_ibv_create_cq_ex(struct ibv_context *context, { struct verbs_context *vctx = verbs_get_ctx(context); struct ibv_cq_ex *cq; + int err = 0; if (cq_attr->wc_flags & ~IBV_CREATE_CQ_SUP_WC_FLAGS) { errno = EOPNOTSUPP; @@ -165,12 +186,20 @@ __lib_ibv_create_cq_ex(struct ibv_context *context, } cq = vctx->priv->create_cq_ex(context, cq_attr); + if (!cq) + return NULL; - if (cq) - verbs_init_cq(ibv_cq_ex_to_cq(cq), context, - cq_attr->channel, cq_attr->cq_context); + err = verbs_init_cq(ibv_cq_ex_to_cq(cq), context, + cq_attr->channel, cq_attr->cq_context); + if (err) + goto err; return cq; + +err: + context->ops.destroy_cq(ibv_cq_ex_to_cq(cq)); + + return NULL; } struct ibv_context *__ibv_open_device(struct ibv_device *device) @@ -198,6 +227,11 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) context = verbs_device->ops->alloc_context(device, cmd_fd); if (!context) goto err; + + if (pthread_mutex_init(&context->mutex, NULL)) { + verbs_device->ops->free_context(context); + goto err; + } } else { struct verbs_ex_private *priv; @@ -212,8 +246,7 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) priv = calloc(1, sizeof(*priv)); if (!priv) { errno = ENOMEM; - free(context_ex); - goto err; + goto err_context; } context_ex->priv = priv; @@ -221,9 +254,12 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) context_ex->sz = sizeof(*context_ex); context = &context_ex->context; + if (pthread_mutex_init(&context->mutex, NULL)) + goto verbs_err; + ret = verbs_device->ops->init_context(verbs_device, context, cmd_fd); if (ret) - goto verbs_err; + goto err_mutex; /* * In order to maintain backward/forward binary compatibility * with apps compiled against libibverbs-1.1.8 that use the @@ -247,12 +283,14 @@ struct ibv_context *__ibv_open_device(struct ibv_device *device) context->device = device; context->cmd_fd = cmd_fd; - pthread_mutex_init(&context->mutex, NULL); return context; +err_mutex: + pthread_mutex_destroy(&context->mutex); verbs_err: free(context_ex->priv); +err_context: free(context_ex); err: close(cmd_fd); @@ -267,6 +305,7 @@ int __ibv_close_device(struct ibv_context *context) struct verbs_context *context_ex; struct verbs_device *verbs_device = verbs_get_device(context->device); + pthread_mutex_destroy(&context->mutex); context_ex = verbs_get_ctx(context); if (context_ex) { verbs_device->ops->uninit_context(verbs_device, context); @@ -393,3 +432,31 @@ void __ibv_ack_async_event(struct ibv_async_event *event) } } default_symver(__ibv_ack_async_event, ibv_ack_async_event); + +int __ibv_init_wq(struct ibv_wq *wq) +{ + int err = 0; + wq->events_completed = 0; + err = pthread_mutex_init(&wq->mutex, NULL); + if (err) + return err; + + err = pthread_cond_init(&wq->cond, NULL); + if (err) + goto err; + + return err; + +err: + pthread_mutex_destroy(&wq->mutex); + + return err; +} +default_symver(__ibv_init_wq, ibv_init_wq); + +void __ibv_cleanup_wq(struct ibv_wq *wq) +{ + pthread_cond_destroy(&wq->mutex); + pthread_mutex_destroy(&wq->mutex); +} +default_symver(__ibv_cleanup_wq, ibv_cleanup_wq); diff --git a/contrib/ofed/libibverbs/driver.h b/contrib/ofed/libibverbs/driver.h index ec87afd7f11e..60824bf27e37 100644 --- a/contrib/ofed/libibverbs/driver.h +++ b/contrib/ofed/libibverbs/driver.h @@ -130,9 +130,12 @@ verbs_get_device(const struct ibv_device *dev) typedef struct verbs_device *(*verbs_driver_init_func)(const char *uverbs_sys_path, int abi_version); void verbs_register_driver(const char *name, verbs_driver_init_func init_func); -void verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context, +int verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context, struct ibv_comp_channel *channel, void *cq_context); +void verbs_cleanup_cq(struct ibv_cq *cq); +int ibv_init_wq(struct ibv_wq *wq); +void ibv_cleanup_wq(struct ibv_wq *wq); int ibv_cmd_get_context(struct ibv_context *context, struct ibv_get_context *cmd, size_t cmd_size, struct ibv_get_context_resp *resp, diff --git a/contrib/ofed/libibverbs/libibverbs.map b/contrib/ofed/libibverbs/libibverbs.map index b49c09a06ce3..d94743389f47 100644 --- a/contrib/ofed/libibverbs/libibverbs.map +++ b/contrib/ofed/libibverbs/libibverbs.map @@ -136,4 +136,7 @@ IBVERBS_PRIVATE_14 { ibv_query_gid_type; verbs_register_driver; verbs_init_cq; + verbs_cleanup_cq; + ibv_init_wq; + ibv_cleanup_wq; }; diff --git a/contrib/ofed/libibverbs/verbs.c b/contrib/ofed/libibverbs/verbs.c index aec8706fd0cc..5c23406e69e7 100644 --- a/contrib/ofed/libibverbs/verbs.c +++ b/contrib/ofed/libibverbs/verbs.c @@ -455,13 +455,23 @@ struct ibv_cq *__ibv_create_cq(struct ibv_context *context, int cqe, void *cq_co struct ibv_comp_channel *channel, int comp_vector) { struct ibv_cq *cq; + int err = 0; cq = context->ops.create_cq(context, cqe, channel, comp_vector); - if (cq) - verbs_init_cq(cq, context, channel, cq_context); + if (!cq) + return NULL; + + err = verbs_init_cq(cq, context, channel, cq_context); + if (err) + goto err; return cq; + +err: + context->ops.destroy_cq(cq); + + return NULL; } default_symver(__ibv_create_cq, ibv_create_cq); @@ -529,16 +539,26 @@ struct ibv_srq *__ibv_create_srq(struct ibv_pd *pd, return NULL; srq = pd->context->ops.create_srq(pd, srq_init_attr); - if (srq) { - srq->context = pd->context; - srq->srq_context = srq_init_attr->srq_context; - srq->pd = pd; - srq->events_completed = 0; - pthread_mutex_init(&srq->mutex, NULL); - pthread_cond_init(&srq->cond, NULL); - } + if (!srq) + return NULL; + + srq->context = pd->context; + srq->srq_context = srq_init_attr->srq_context; + srq->pd = pd; + srq->events_completed = 0; + if (pthread_mutex_init(&srq->mutex, NULL)) + goto err; + if (pthread_cond_init(&srq->cond, NULL)) + goto err_mutex; return srq; + +err_mutex: + pthread_mutex_destroy(&srq->mutex); +err: + pd->context->ops.destroy_srq(srq); + + return NULL; } default_symver(__ibv_create_srq, ibv_create_srq); @@ -558,6 +578,8 @@ default_symver(__ibv_query_srq, ibv_query_srq); int __ibv_destroy_srq(struct ibv_srq *srq) { + pthread_cond_destroy(&srq->cond); + pthread_mutex_destroy(&srq->mutex); return srq->context->ops.destroy_srq(srq); } default_symver(__ibv_destroy_srq, ibv_destroy_srq); diff --git a/contrib/ofed/libibverbs/verbs.h b/contrib/ofed/libibverbs/verbs.h index 498275561280..bea817e36fc9 100644 --- a/contrib/ofed/libibverbs/verbs.h +++ b/contrib/ofed/libibverbs/verbs.h @@ -2166,11 +2166,8 @@ static inline struct ibv_wq *ibv_create_wq(struct ibv_context *context, } wq = vctx->create_wq(context, wq_init_attr); - if (wq) { + if (wq) wq->events_completed = 0; - pthread_mutex_init(&wq->mutex, NULL); - pthread_cond_init(&wq->cond, NULL); - } return wq; } diff --git a/contrib/ofed/libmlx4/mlx4.c b/contrib/ofed/libmlx4/mlx4.c index 229c2670b5ed..db8a07d48381 100644 --- a/contrib/ofed/libmlx4/mlx4.c +++ b/contrib/ofed/libmlx4/mlx4.c @@ -147,6 +147,7 @@ static int mlx4_init_context(struct verbs_device *v_device, struct ibv_get_context cmd; struct mlx4_alloc_ucontext_resp resp; int i; + int ret; struct mlx4_alloc_ucontext_resp_v3 resp_v3; __u16 bf_reg_size; struct mlx4_device *dev = to_mdev(&v_device->device); @@ -185,15 +186,22 @@ static int mlx4_init_context(struct verbs_device *v_device, for (i = 0; i < MLX4_PORTS_NUM; ++i) context->port_query_cache[i].valid = 0; - pthread_mutex_init(&context->qp_table_mutex, NULL); + ret = pthread_mutex_init(&context->qp_table_mutex, NULL); + if (ret) + return ret; for (i = 0; i < MLX4_QP_TABLE_SIZE; ++i) context->qp_table[i].refcnt = 0; for (i = 0; i < MLX4_NUM_DB_TYPE; ++i) context->db_list[i] = NULL; - mlx4_init_xsrq_table(&context->xsrq_table, context->num_qps); - pthread_mutex_init(&context->db_list_mutex, NULL); + ret = mlx4_init_xsrq_table(&context->xsrq_table, context->num_qps); + if (ret) + goto err; + + ret = pthread_mutex_init(&context->db_list_mutex, NULL); + if (ret) + goto err_xsrq; context->uar = mmap(NULL, dev->page_size, PROT_WRITE, MAP_SHARED, cmd_fd, 0); @@ -212,14 +220,18 @@ static int mlx4_init_context(struct verbs_device *v_device, } else { context->bf_buf_size = bf_reg_size / 2; context->bf_offset = 0; - pthread_spin_init(&context->bf_lock, PTHREAD_PROCESS_PRIVATE); + ret = pthread_spin_init(&context->bf_lock, PTHREAD_PROCESS_PRIVATE); + if (ret) + goto err_db_list; } } else { context->bf_page = NULL; context->bf_buf_size = 0; } - pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); + ret = pthread_spin_init(&context->uar_lock, PTHREAD_PROCESS_PRIVATE); + if (ret) + goto err_bf_lock; ibv_ctx->ops = mlx4_ctx_ops; context->hca_core_clock = NULL; @@ -248,6 +260,17 @@ static int mlx4_init_context(struct verbs_device *v_device, return 0; +err_bf_lock: + if (context->bf_buf_size) + pthread_spin_destroy(&context->bf_lock); +err_db_list: + pthread_mutex_destroy(&context->db_list_mutex); +err_xsrq: + mlx4_cleanup_xsrq_table(&context->xsrq_table); +err: + pthread_mutex_destroy(&context->qp_table_mutex); + + return ret; } static void mlx4_uninit_context(struct verbs_device *v_device, @@ -255,6 +278,12 @@ static void mlx4_uninit_context(struct verbs_device *v_device, { struct mlx4_context *context = to_mctx(ibv_ctx); + pthread_mutex_destroy(&context->qp_table_mutex); + mlx4_cleanup_xsrq_table(&context->xsrq_table); + pthread_mutex_destroy(&context->db_list_mutex); + pthread_spin_destroy(&context->bf_lock); + pthread_spin_destroy(&context->uar_lock); + munmap(context->uar, to_mdev(&v_device->device)->page_size); if (context->bf_page) munmap(context->bf_page, to_mdev(&v_device->device)->page_size); diff --git a/contrib/ofed/libmlx4/mlx4.h b/contrib/ofed/libmlx4/mlx4.h index 864ef9eccc60..5b3dd547b5fe 100644 --- a/contrib/ofed/libmlx4/mlx4.h +++ b/contrib/ofed/libmlx4/mlx4.h @@ -414,7 +414,8 @@ int mlx4_destroy_srq(struct ibv_srq *srq); int mlx4_destroy_xrc_srq(struct ibv_srq *srq); int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, struct mlx4_srq *srq); -void mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size); +void mlx4_cleanup_xsrq_table(struct mlx4_xsrq_table *xsrq_table); +int mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size); struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn); int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn, struct mlx4_srq *srq); diff --git a/contrib/ofed/libmlx4/srq.c b/contrib/ofed/libmlx4/srq.c index b8d25bb343da..da709c630450 100644 --- a/contrib/ofed/libmlx4/srq.c +++ b/contrib/ofed/libmlx4/srq.c @@ -172,14 +172,20 @@ int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, return 0; } -void mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size) +void mlx4_cleanup_xsrq_table(struct mlx4_xsrq_table *xsrq_table) { + pthread_mutex_destroy(&xsrq_table->mutex); +} + +int mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size) +{ + int ret; memset(xsrq_table, 0, sizeof *xsrq_table); xsrq_table->num_xsrq = size; xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS; xsrq_table->mask = (1 << xsrq_table->shift) - 1; - pthread_mutex_init(&xsrq_table->mutex, NULL); + return pthread_mutex_init(&xsrq_table->mutex, NULL); } struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn) @@ -257,7 +263,7 @@ struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context, srq->ext_srq = 1; if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq)) - goto err; + goto err_spl; srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ); if (!srq->db) @@ -290,6 +296,8 @@ err_db: err_free: free(srq->wrid); mlx4_free_buf(&srq->buf); +err_spl: + pthread_spin_destroy(&srq->lock); *** 468 LINES SKIPPED ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202309191412.38JEC4go072915>