From owner-svn-src-projects@FreeBSD.ORG Mon Jan 3 05:33:52 2011 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id D2124106566B; Mon, 3 Jan 2011 05:33:52 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id BD8748FC12; Mon, 3 Jan 2011 05:33:52 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id p035Xq5w075899; Mon, 3 Jan 2011 05:33:52 GMT (envelope-from jeff@svn.freebsd.org) Received: (from jeff@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id p035Xq4m075887; Mon, 3 Jan 2011 05:33:52 GMT (envelope-from jeff@svn.freebsd.org) Message-Id: <201101030533.p035Xq4m075887@svn.freebsd.org> From: Jeff Roberson Date: Mon, 3 Jan 2011 05:33:52 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r216908 - in projects/ofed/base/contrib/ofed: libibverbs libibverbs/examples libibverbs/fixes libibverbs/include/infiniband libibverbs/man libibverbs/src libmlx4 libmlx4/fixes libmlx4/src X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 03 Jan 2011 05:33:52 -0000 Author: jeff Date: Mon Jan 3 05:33:52 2011 New Revision: 216908 URL: http://svn.freebsd.org/changeset/base/216908 Log: - Update libibverbs and libmlx4. This includes XRC support. Sponsored by: Isilon Systems, iX Systems, and Panasas. Modified: projects/ofed/base/contrib/ofed/libibverbs/Makefile.am projects/ofed/base/contrib/ofed/libibverbs/configure.in projects/ofed/base/contrib/ofed/libibverbs/examples/asyncwatch.c projects/ofed/base/contrib/ofed/libibverbs/examples/devinfo.c projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.c projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.h projects/ofed/base/contrib/ofed/libibverbs/examples/rc_pingpong.c projects/ofed/base/contrib/ofed/libibverbs/examples/srq_pingpong.c projects/ofed/base/contrib/ofed/libibverbs/examples/uc_pingpong.c projects/ofed/base/contrib/ofed/libibverbs/examples/ud_pingpong.c projects/ofed/base/contrib/ofed/libibverbs/fixes/qpt_raw_eth.patch projects/ofed/base/contrib/ofed/libibverbs/fixes/series projects/ofed/base/contrib/ofed/libibverbs/include/infiniband/driver.h projects/ofed/base/contrib/ofed/libibverbs/include/infiniband/kern-abi.h projects/ofed/base/contrib/ofed/libibverbs/include/infiniband/verbs.h projects/ofed/base/contrib/ofed/libibverbs/libibverbs.spec.in projects/ofed/base/contrib/ofed/libibverbs/man/ibv_create_qp.3 projects/ofed/base/contrib/ofed/libibverbs/man/ibv_create_srq.3 projects/ofed/base/contrib/ofed/libibverbs/man/ibv_post_send.3 projects/ofed/base/contrib/ofed/libibverbs/man/ibv_query_port.3 projects/ofed/base/contrib/ofed/libibverbs/src/cmd.c projects/ofed/base/contrib/ofed/libibverbs/src/device.c projects/ofed/base/contrib/ofed/libibverbs/src/libibverbs.map projects/ofed/base/contrib/ofed/libibverbs/src/verbs.c projects/ofed/base/contrib/ofed/libmlx4/configure.in projects/ofed/base/contrib/ofed/libmlx4/fixes/rocee_add_support.patch projects/ofed/base/contrib/ofed/libmlx4/fixes/series projects/ofed/base/contrib/ofed/libmlx4/src/cq.c projects/ofed/base/contrib/ofed/libmlx4/src/mlx4-abi.h projects/ofed/base/contrib/ofed/libmlx4/src/mlx4.c projects/ofed/base/contrib/ofed/libmlx4/src/mlx4.h projects/ofed/base/contrib/ofed/libmlx4/src/qp.c projects/ofed/base/contrib/ofed/libmlx4/src/srq.c projects/ofed/base/contrib/ofed/libmlx4/src/verbs.c projects/ofed/base/contrib/ofed/libmlx4/src/wqe.h Modified: projects/ofed/base/contrib/ofed/libibverbs/Makefile.am ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/Makefile.am Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/Makefile.am Mon Jan 3 05:33:52 2011 (r216908) @@ -44,16 +44,19 @@ man_MANS = man/ibv_asyncwatch.1 man/ibv_ man/ibv_srq_pingpong.1 man/ibv_alloc_pd.3 man/ibv_attach_mcast.3 \ man/ibv_create_ah.3 man/ibv_create_ah_from_wc.3 \ man/ibv_create_comp_channel.3 man/ibv_create_cq.3 \ - man/ibv_create_qp.3 man/ibv_create_srq.3 man/ibv_event_type_str.3 \ + man/ibv_create_qp.3 man/ibv_create_srq.3 \ + man/ibv_create_xrc_rcv_qp.3 man/ibv_event_type_str.3 \ man/ibv_fork_init.3 man/ibv_get_async_event.3 \ man/ibv_get_cq_event.3 man/ibv_get_device_guid.3 \ man/ibv_get_device_list.3 man/ibv_get_device_name.3 \ - man/ibv_modify_qp.3 man/ibv_modify_srq.3 man/ibv_open_device.3 \ + man/ibv_modify_qp.3 man/ibv_modify_srq.3 man/ibv_modify_xrc_rcv_qp.3 \ + man/ibv_open_device.3 man/ibv_open_xrc_domain.3 \ man/ibv_poll_cq.3 man/ibv_post_recv.3 man/ibv_post_send.3 \ man/ibv_post_srq_recv.3 man/ibv_query_device.3 man/ibv_query_gid.3 \ man/ibv_query_pkey.3 man/ibv_query_port.3 man/ibv_query_qp.3 \ - man/ibv_query_srq.3 man/ibv_rate_to_mult.3 man/ibv_reg_mr.3 \ - man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 + man/ibv_query_srq.3 man/ibv_query_xrc_rcv_qp.3 \ + man/ibv_rate_to_mult.3 man/ibv_reg_mr.3 man/ibv_reg_xrc_rcv_qp.3 \ + man/ibv_req_notify_cq.3 man/ibv_resize_cq.3 man/verbs.7 DEBIAN = debian/changelog debian/compat debian/control debian/copyright \ debian/ibverbs-utils.install debian/libibverbs1.install \ @@ -74,6 +77,8 @@ install-data-hook: $(RM) ibv_ack_async_event.3 && \ $(RM) ibv_ack_cq_events.3 && \ $(RM) ibv_close_device.3 && \ + $(RM) ibv_close_xrc_domain.3 && \ + $(RM) ibv_create_xrc_srq.3 && \ $(RM) ibv_dealloc_pd.3 && \ $(RM) ibv_dereg_mr.3 && \ $(RM) ibv_destroy_ah.3 && \ @@ -84,12 +89,15 @@ install-data-hook: $(RM) ibv_detach_mcast.3 && \ $(RM) ibv_free_device_list.3 && \ $(RM) ibv_init_ah_from_wc.3 && \ + $(RM) ibv_unreg_xrc_rcv_qp.3 && \ $(RM) mult_to_ibv_rate.3 && \ $(RM) ibv_node_type_str.3 && \ $(RM) ibv_port_state_str.3 && \ $(LN_S) ibv_get_async_event.3 ibv_ack_async_event.3 && \ $(LN_S) ibv_get_cq_event.3 ibv_ack_cq_events.3 && \ $(LN_S) ibv_open_device.3 ibv_close_device.3 && \ + $(LN_S) ibv_open_xrc_domain.3 ibv_close_xrc_domain.3 && \ + $(LN_S) ibv_create_srq.3 ibv_create_xrc_srq.3 && \ $(LN_S) ibv_alloc_pd.3 ibv_dealloc_pd.3 && \ $(LN_S) ibv_reg_mr.3 ibv_dereg_mr.3 && \ $(LN_S) ibv_create_ah.3 ibv_destroy_ah.3 && \ @@ -100,6 +108,7 @@ install-data-hook: $(LN_S) ibv_attach_mcast.3 ibv_detach_mcast.3 && \ $(LN_S) ibv_get_device_list.3 ibv_free_device_list.3 && \ $(LN_S) ibv_create_ah_from_wc.3 ibv_init_ah_from_wc.3 && \ + $(LN_S) ibv_reg_xrc_rcv_qp.3 ibv_unreg_xrc_rcv_qp.3 && \ $(LN_S) ibv_rate_to_mult.3 mult_to_ibv_rate.3 && \ $(LN_S) ibv_event_type_str.3 ibv_node_type_str.3 && \ $(LN_S) ibv_event_type_str.3 ibv_port_state_str.3 Modified: projects/ofed/base/contrib/ofed/libibverbs/configure.in ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/configure.in Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/configure.in Mon Jan 3 05:33:52 2011 (r216908) @@ -13,6 +13,7 @@ AC_PROG_CC AC_GNU_SOURCE AC_PROG_LN_S +AC_PROG_LIBTOOL LT_INIT AC_ARG_WITH([valgrind], Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/asyncwatch.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/asyncwatch.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/asyncwatch.c Mon Jan 3 05:33:52 2011 (r216908) @@ -57,6 +57,8 @@ static const char *event_name_str(enum i return "IBV_EVENT_SM_CHANGE"; case IBV_EVENT_CLIENT_REREGISTER: return "IBV_EVENT_CLIENT_REREGISTER"; + case IBV_EVENT_GID_CHANGE: + return "IBV_EVENT_GID_CHANGE"; case IBV_EVENT_CQ_ERR: case IBV_EVENT_QP_FATAL: Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/devinfo.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/devinfo.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/devinfo.c Mon Jan 3 05:33:52 2011 (r216908) @@ -184,6 +184,19 @@ static int print_all_port_gids(struct ib return rc; } +static const char *link_layer_str(uint8_t link_layer) +{ + switch (link_layer) { + case IBV_LINK_LAYER_UNSPECIFIED: + case IBV_LINK_LAYER_INFINIBAND: + return "IB"; + case IBV_LINK_LAYER_ETHERNET: + return "Ethernet"; + default: + return "Unknown"; + } +} + static int print_hca_cap(struct ibv_device *ib_dev, uint8_t ib_port) { struct ibv_context *ctx; @@ -284,6 +297,7 @@ static int print_hca_cap(struct ibv_devi printf("\t\t\tsm_lid:\t\t\t%d\n", port_attr.sm_lid); printf("\t\t\tport_lid:\t\t%d\n", port_attr.lid); printf("\t\t\tport_lmc:\t\t0x%02x\n", port_attr.lmc); + printf("\t\t\tlink_layer:\t\t%s\n", link_layer_str(port_attr.link_layer)); if (verbose) { printf("\t\t\tmax_msg_sz:\t\t0x%x\n", port_attr.max_msg_sz); Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.c Mon Jan 3 05:33:52 2011 (r216908) @@ -31,6 +31,10 @@ */ #include "pingpong.h" +#include +#include +#include +#include enum ibv_mtu pp_mtu_to_enum(int mtu) { @@ -53,3 +57,30 @@ uint16_t pp_get_local_lid(struct ibv_con return attr.lid; } + +int pp_get_port_info(struct ibv_context *context, int port, + struct ibv_port_attr *attr) +{ + return ibv_query_port(context, port, attr); +} + +void wire_gid_to_gid(const char *wgid, union ibv_gid *gid) +{ + char tmp[9]; + uint32_t v32; + int i; + + for (tmp[8] = 0, i = 0; i < 4; ++i) { + memcpy(tmp, wgid + i * 8, 8); + sscanf(tmp, "%x", &v32); + *(uint32_t *)(&gid->raw[i * 4]) = ntohl(v32); + } +} + +void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]) +{ + int i; + + for (i = 0; i < 4; ++i) + sprintf(&wgid[i * 8], "%08x", htonl(*(uint32_t *)(gid->raw + i * 4))); +} Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.h ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.h Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/pingpong.h Mon Jan 3 05:33:52 2011 (r216908) @@ -37,5 +37,9 @@ enum ibv_mtu pp_mtu_to_enum(int mtu); uint16_t pp_get_local_lid(struct ibv_context *context, int port); +int pp_get_port_info(struct ibv_context *context, int port, + struct ibv_port_attr *attr); +void wire_gid_to_gid(const char *wgid, union ibv_gid *gid); +void gid_to_wire_gid(const union ibv_gid *gid, char wgid[]); #endif /* IBV_PINGPONG_H */ Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/rc_pingpong.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/rc_pingpong.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/rc_pingpong.c Mon Jan 3 05:33:52 2011 (r216908) @@ -67,17 +67,19 @@ struct pingpong_context { int size; int rx_depth; int pending; + struct ibv_port_attr portinfo; }; struct pingpong_dest { int lid; int qpn; int psn; + union ibv_gid gid; }; static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, enum ibv_mtu mtu, int sl, - struct pingpong_dest *dest) + struct pingpong_dest *dest, int sgid_idx) { struct ibv_qp_attr attr = { .qp_state = IBV_QPS_RTR, @@ -94,6 +96,13 @@ static int pp_connect_ctx(struct pingpon .port_num = port } }; + + if (dest->gid.global.interface_id) { + attr.ah_attr.is_global = 1; + attr.ah_attr.grh.hop_limit = 1; + attr.ah_attr.grh.dgid = dest->gid; + attr.ah_attr.grh.sgid_index = sgid_idx; + } if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE | IBV_QP_AV | @@ -135,10 +144,11 @@ static struct pingpong_dest *pp_client_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int sockfd = -1; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -169,7 +179,8 @@ static struct pingpong_dest *pp_client_e return NULL; } - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); + gid_to_wire_gid(&my_dest->gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); if (write(sockfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); goto out; @@ -187,7 +198,8 @@ static struct pingpong_dest *pp_client_e if (!rem_dest) goto out; - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); + wire_gid_to_gid(gid, &rem_dest->gid); out: close(sockfd); @@ -197,7 +209,8 @@ out: static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, int ib_port, enum ibv_mtu mtu, int port, int sl, - const struct pingpong_dest *my_dest) + const struct pingpong_dest *my_dest, + int sgid_idx) { struct addrinfo *res, *t; struct addrinfo hints = { @@ -206,10 +219,11 @@ static struct pingpong_dest *pp_server_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int sockfd = -1, connfd; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -263,16 +277,19 @@ static struct pingpong_dest *pp_server_e if (!rem_dest) goto out; - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); + wire_gid_to_gid(gid, &rem_dest->gid); - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) { + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { fprintf(stderr, "Couldn't connect to remote QP\n"); free(rem_dest); rem_dest = NULL; goto out; } - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); + + gid_to_wire_gid(&my_dest->gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); if (write(connfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); free(rem_dest); @@ -289,11 +306,11 @@ out: static struct pingpong_context *pp_init_ctx(struct ibv_device *ib_dev, int size, int rx_depth, int port, - int use_event) + int use_event, int is_server) { struct pingpong_context *ctx; - ctx = malloc(sizeof *ctx); + ctx = calloc(1, sizeof *ctx); if (!ctx) return NULL; @@ -306,7 +323,7 @@ static struct pingpong_context *pp_init_ return NULL; } - memset(ctx->buf, 0, size); + memset(ctx->buf, 0x7b + is_server, size); ctx->context = ibv_open_device(ib_dev); if (!ctx->context) { @@ -481,6 +498,7 @@ static void usage(const char *argv0) printf(" -n, --iters= number of exchanges (default 1000)\n"); printf(" -l, --sl= service level value\n"); printf(" -e, --events sleep on CQ events (default poll)\n"); + printf(" -g, --gid-idx= local port gid index\n"); } int main(int argc, char *argv[]) @@ -504,6 +522,8 @@ int main(int argc, char *argv[]) int rcnt, scnt; int num_cq_events = 0; int sl = 0; + int gidx = -1; + char gid[33]; srand48(getpid() * time(NULL)); @@ -520,10 +540,11 @@ int main(int argc, char *argv[]) { .name = "iters", .has_arg = 1, .val = 'n' }, { .name = "sl", .has_arg = 1, .val = 'l' }, { .name = "events", .has_arg = 0, .val = 'e' }, + { .name = "gid-idx", .has_arg = 1, .val = 'g' }, { 0 } }; - c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL); + c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); if (c == -1) break; @@ -576,6 +597,10 @@ int main(int argc, char *argv[]) ++use_event; break; + case 'g': + gidx = strtol(optarg, NULL, 0); + break; + default: usage(argv[0]); return 1; @@ -615,7 +640,7 @@ int main(int argc, char *argv[]) } } - ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event); + ctx = pp_init_ctx(ib_dev, size, rx_depth, ib_port, use_event, !servername); if (!ctx) return 1; @@ -631,30 +656,47 @@ int main(int argc, char *argv[]) return 1; } - my_dest.lid = pp_get_local_lid(ctx->context, ib_port); - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = lrand48() & 0xffffff; - if (!my_dest.lid) { + + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { + fprintf(stderr, "Couldn't get port info\n"); + return 1; + } + + my_dest.lid = ctx->portinfo.lid; + if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { fprintf(stderr, "Couldn't get local LID\n"); return 1; } - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", - my_dest.lid, my_dest.qpn, my_dest.psn); + if (gidx >= 0) { + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); + return 1; + } + } else + memset(&my_dest.gid, 0, sizeof my_dest.gid); + + my_dest.qpn = ctx->qp->qp_num; + my_dest.psn = lrand48() & 0xffffff; + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + my_dest.lid, my_dest.qpn, my_dest.psn, gid); + if (servername) rem_dest = pp_client_exch_dest(servername, port, &my_dest); else - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest); + rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); if (!rem_dest) return 1; - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", - rem_dest->lid, rem_dest->qpn, rem_dest->psn); + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); if (servername) - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest)) + if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) return 1; ctx->pending = PINGPONG_RECV_WRID; @@ -706,6 +748,7 @@ int main(int argc, char *argv[]) fprintf(stderr, "poll CQ failed %d\n", ne); return 1; } + } while (!use_event && ne < 1); for (i = 0; i < ne; ++i) { Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/srq_pingpong.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/srq_pingpong.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/srq_pingpong.c Mon Jan 3 05:33:52 2011 (r216908) @@ -71,17 +71,19 @@ struct pingpong_context { int num_qp; int rx_depth; int pending[MAX_QP]; + struct ibv_port_attr portinfo; }; struct pingpong_dest { int lid; int qpn; int psn; + union ibv_gid gid; }; static int pp_connect_ctx(struct pingpong_context *ctx, int port, enum ibv_mtu mtu, int sl, const struct pingpong_dest *my_dest, - const struct pingpong_dest *dest) + const struct pingpong_dest *dest, int sgid_idx) { int i; @@ -101,6 +103,13 @@ static int pp_connect_ctx(struct pingpon .port_num = port } }; + + if (dest->gid.global.interface_id) { + attr.ah_attr.is_global = 1; + attr.ah_attr.grh.hop_limit = 1; + attr.ah_attr.grh.dgid = dest->gid; + attr.ah_attr.grh.sgid_index = sgid_idx; + } if (ibv_modify_qp(ctx->qp[i], &attr, IBV_QP_STATE | IBV_QP_AV | @@ -143,12 +152,13 @@ static struct pingpong_dest *pp_client_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int r; int i; int sockfd = -1; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -180,7 +190,8 @@ static struct pingpong_dest *pp_client_e } for (i = 0; i < MAX_QP; ++i) { - sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); + gid_to_wire_gid(&my_dest[i].gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); if (write(sockfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); goto out; @@ -204,8 +215,9 @@ static struct pingpong_dest *pp_client_e n += r; } - sscanf(msg, "%x:%x:%x", - &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn); + sscanf(msg, "%x:%x:%x:%s", + &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid); + wire_gid_to_gid(gid, &rem_dest[i].gid); } write(sockfd, "done", sizeof "done"); @@ -218,7 +230,8 @@ out: static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, int ib_port, enum ibv_mtu mtu, int port, int sl, - const struct pingpong_dest *my_dest) + const struct pingpong_dest *my_dest, + int sgid_idx) { struct addrinfo *res, *t; struct addrinfo hints = { @@ -227,12 +240,13 @@ static struct pingpong_dest *pp_server_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int r; int i; int sockfd = -1, connfd; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -292,11 +306,12 @@ static struct pingpong_dest *pp_server_e n += r; } - sscanf(msg, "%x:%x:%x", - &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn); + sscanf(msg, "%x:%x:%x:%s", + &rem_dest[i].lid, &rem_dest[i].qpn, &rem_dest[i].psn, gid); + wire_gid_to_gid(gid, &rem_dest[i].gid); } - if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) { + if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, sgid_idx)) { fprintf(stderr, "Couldn't connect to remote QP\n"); free(rem_dest); rem_dest = NULL; @@ -304,7 +319,8 @@ static struct pingpong_dest *pp_server_e } for (i = 0; i < MAX_QP; ++i) { - sprintf(msg, "%04x:%06x:%06x", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); + gid_to_wire_gid(&my_dest[i].gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); if (write(connfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); free(rem_dest); @@ -327,7 +343,7 @@ static struct pingpong_context *pp_init_ struct pingpong_context *ctx; int i; - ctx = malloc(sizeof *ctx); + ctx = calloc(1, sizeof *ctx); if (!ctx) return NULL; @@ -551,6 +567,7 @@ static void usage(const char *argv0) printf(" -n, --iters= number of exchanges per QP(default 1000)\n"); printf(" -l, --sl= service level value\n"); printf(" -e, --events sleep on CQ events (default poll)\n"); + printf(" -g, --gid-idx= local port gid index\n"); } int main(int argc, char *argv[]) @@ -578,6 +595,8 @@ int main(int argc, char *argv[]) int i; int num_cq_events = 0; int sl = 0; + int gidx = -1; + char gid[33]; srand48(getpid() * time(NULL)); @@ -595,10 +614,11 @@ int main(int argc, char *argv[]) { .name = "iters", .has_arg = 1, .val = 'n' }, { .name = "sl", .has_arg = 1, .val = 'l' }, { .name = "events", .has_arg = 0, .val = 'e' }, + { .name = "gid-idx", .has_arg = 1, .val = 'g' }, { 0 } }; - c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:e", long_options, NULL); + c = getopt_long(argc, argv, "p:d:i:s:m:q:r:n:l:eg:", long_options, NULL); if (c == -1) break; @@ -655,6 +675,10 @@ int main(int argc, char *argv[]) ++use_event; break; + case 'g': + gidx = strtol(optarg, NULL, 0); + break; + default: usage(argv[0]); return 1; @@ -722,33 +746,50 @@ int main(int argc, char *argv[]) memset(my_dest, 0, sizeof my_dest); + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { + fprintf(stderr, "Couldn't get port info\n"); + return 1; + } for (i = 0; i < num_qp; ++i) { my_dest[i].qpn = ctx->qp[i]->qp_num; my_dest[i].psn = lrand48() & 0xffffff; - my_dest[i].lid = pp_get_local_lid(ctx->context, ib_port); - if (!my_dest[i].lid) { + my_dest[i].lid = ctx->portinfo.lid; + if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest[i].lid) { fprintf(stderr, "Couldn't get local LID\n"); return 1; } - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", - my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn); + if (gidx >= 0) { + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest[i].gid)) { + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); + return 1; + } + } else + memset(&my_dest[i].gid, 0, sizeof my_dest[i].gid); + + inet_ntop(AF_INET6, &my_dest[i].gid, gid, sizeof gid); + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + my_dest[i].lid, my_dest[i].qpn, my_dest[i].psn, gid); } if (servername) rem_dest = pp_client_exch_dest(servername, port, my_dest); else - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest); + rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, my_dest, gidx); if (!rem_dest) return 1; - for (i = 0; i < num_qp; ++i) - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", - rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn); + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); + + for (i = 0; i < num_qp; ++i) { + inet_ntop(AF_INET6, &rem_dest[i].gid, gid, sizeof gid); + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + rem_dest[i].lid, rem_dest[i].qpn, rem_dest[i].psn, gid); + } if (servername) - if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest)) + if (pp_connect_ctx(ctx, ib_port, mtu, sl, my_dest, rem_dest, gidx)) return 1; if (servername) Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/uc_pingpong.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/uc_pingpong.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/uc_pingpong.c Mon Jan 3 05:33:52 2011 (r216908) @@ -67,17 +67,19 @@ struct pingpong_context { int size; int rx_depth; int pending; + struct ibv_port_attr portinfo; }; struct pingpong_dest { int lid; int qpn; int psn; + union ibv_gid gid; }; static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, enum ibv_mtu mtu, int sl, - struct pingpong_dest *dest) + struct pingpong_dest *dest, int sgid_idx) { struct ibv_qp_attr attr = { .qp_state = IBV_QPS_RTR, @@ -92,6 +94,14 @@ static int pp_connect_ctx(struct pingpon .port_num = port } }; + + if (dest->gid.global.interface_id) { + attr.ah_attr.is_global = 1; + attr.ah_attr.grh.hop_limit = 1; + attr.ah_attr.grh.dgid = dest->gid; + attr.ah_attr.grh.sgid_index = sgid_idx; + } + if (ibv_modify_qp(ctx->qp, &attr, IBV_QP_STATE | IBV_QP_AV | @@ -123,10 +133,11 @@ static struct pingpong_dest *pp_client_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int sockfd = -1; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -157,7 +168,8 @@ static struct pingpong_dest *pp_client_e return NULL; } - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); + gid_to_wire_gid(&my_dest->gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); if (write(sockfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); goto out; @@ -175,7 +187,8 @@ static struct pingpong_dest *pp_client_e if (!rem_dest) goto out; - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); + wire_gid_to_gid(gid, &rem_dest->gid); out: close(sockfd); @@ -185,7 +198,8 @@ out: static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, int ib_port, enum ibv_mtu mtu, int port, int sl, - const struct pingpong_dest *my_dest) + const struct pingpong_dest *my_dest, + int sgid_idx) { struct addrinfo *res, *t; struct addrinfo hints = { @@ -194,10 +208,11 @@ static struct pingpong_dest *pp_server_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int sockfd = -1, connfd; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -251,16 +266,18 @@ static struct pingpong_dest *pp_server_e if (!rem_dest) goto out; - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); + wire_gid_to_gid(gid, &rem_dest->gid); - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest)) { + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, mtu, sl, rem_dest, sgid_idx)) { fprintf(stderr, "Couldn't connect to remote QP\n"); free(rem_dest); rem_dest = NULL; goto out; } - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); + gid_to_wire_gid(&my_dest->gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); if (write(connfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); free(rem_dest); @@ -281,7 +298,7 @@ static struct pingpong_context *pp_init_ { struct pingpong_context *ctx; - ctx = malloc(sizeof *ctx); + ctx = calloc(1, sizeof *ctx); if (!ctx) return NULL; @@ -469,6 +486,7 @@ static void usage(const char *argv0) printf(" -n, --iters= number of exchanges (default 1000)\n"); printf(" -l, --sl= service level value\n"); printf(" -e, --events sleep on CQ events (default poll)\n"); + printf(" -g, --gid-idx= local port gid index\n"); } int main(int argc, char *argv[]) @@ -492,6 +510,8 @@ int main(int argc, char *argv[]) int rcnt, scnt; int num_cq_events = 0; int sl = 0; + int gidx = -1; + char gid[33]; srand48(getpid() * time(NULL)); @@ -508,10 +528,11 @@ int main(int argc, char *argv[]) { .name = "iters", .has_arg = 1, .val = 'n' }, { .name = "sl", .has_arg = 1, .val = 'l' }, { .name = "events", .has_arg = 0, .val = 'e' }, + { .name = "gid-idx", .has_arg = 1, .val = 'g' }, { 0 } }; - c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:e", long_options, NULL); + c = getopt_long(argc, argv, "p:d:i:s:m:r:n:l:eg:", long_options, NULL); if (c == -1) break; @@ -564,6 +585,10 @@ int main(int argc, char *argv[]) ++use_event; break; + case 'g': + gidx = strtol(optarg, NULL, 0); + break; + default: usage(argv[0]); return 1; @@ -619,30 +644,45 @@ int main(int argc, char *argv[]) return 1; } - my_dest.lid = pp_get_local_lid(ctx->context, ib_port); - my_dest.qpn = ctx->qp->qp_num; - my_dest.psn = lrand48() & 0xffffff; - if (!my_dest.lid) { + if (pp_get_port_info(ctx->context, ib_port, &ctx->portinfo)) { + fprintf(stderr, "Couldn't get port info\n"); + return 1; + } + + my_dest.lid = ctx->portinfo.lid; + if (ctx->portinfo.link_layer == IBV_LINK_LAYER_INFINIBAND && !my_dest.lid) { fprintf(stderr, "Couldn't get local LID\n"); return 1; } - printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", - my_dest.lid, my_dest.qpn, my_dest.psn); + if (gidx >= 0) { + if (ibv_query_gid(ctx->context, ib_port, gidx, &my_dest.gid)) { + fprintf(stderr, "Could not get local gid for gid index %d\n", gidx); + return 1; + } + } else + memset(&my_dest.gid, 0, sizeof my_dest.gid); + + my_dest.qpn = ctx->qp->qp_num; + my_dest.psn = lrand48() & 0xffffff; + inet_ntop(AF_INET6, &my_dest.gid, gid, sizeof gid); + printf(" local address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + my_dest.lid, my_dest.qpn, my_dest.psn, gid); if (servername) rem_dest = pp_client_exch_dest(servername, port, &my_dest); else - rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest); + rem_dest = pp_server_exch_dest(ctx, ib_port, mtu, port, sl, &my_dest, gidx); if (!rem_dest) return 1; - printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x\n", - rem_dest->lid, rem_dest->qpn, rem_dest->psn); + inet_ntop(AF_INET6, &rem_dest->gid, gid, sizeof gid); + printf(" remote address: LID 0x%04x, QPN 0x%06x, PSN 0x%06x, GID %s\n", + rem_dest->lid, rem_dest->qpn, rem_dest->psn, gid); if (servername) - if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest)) + if (pp_connect_ctx(ctx, ib_port, my_dest.psn, mtu, sl, rem_dest, gidx)) return 1; ctx->pending = PINGPONG_RECV_WRID; Modified: projects/ofed/base/contrib/ofed/libibverbs/examples/ud_pingpong.c ============================================================================== --- projects/ofed/base/contrib/ofed/libibverbs/examples/ud_pingpong.c Mon Jan 3 05:21:53 2011 (r216907) +++ projects/ofed/base/contrib/ofed/libibverbs/examples/ud_pingpong.c Mon Jan 3 05:33:52 2011 (r216908) @@ -68,16 +68,18 @@ struct pingpong_context { int size; int rx_depth; int pending; + struct ibv_port_attr portinfo; }; struct pingpong_dest { int lid; int qpn; int psn; + union ibv_gid gid; }; static int pp_connect_ctx(struct pingpong_context *ctx, int port, int my_psn, - int sl, struct pingpong_dest *dest) + int sl, struct pingpong_dest *dest, int sgid_idx) { struct ibv_ah_attr ah_attr = { .is_global = 0, @@ -105,6 +107,13 @@ static int pp_connect_ctx(struct pingpon return 1; } + if (dest->gid.global.interface_id) { + ah_attr.is_global = 1; + ah_attr.grh.hop_limit = 1; + ah_attr.grh.dgid = dest->gid; + ah_attr.grh.sgid_index = sgid_idx; + } + ctx->ah = ibv_create_ah(ctx->pd, &ah_attr); if (!ctx->ah) { fprintf(stderr, "Failed to create AH\n"); @@ -123,10 +132,11 @@ static struct pingpong_dest *pp_client_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int sockfd = -1; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -157,7 +167,8 @@ static struct pingpong_dest *pp_client_e return NULL; } - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); + gid_to_wire_gid(&my_dest->gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); if (write(sockfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); goto out; @@ -175,7 +186,8 @@ static struct pingpong_dest *pp_client_e if (!rem_dest) goto out; - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); + wire_gid_to_gid(gid, &rem_dest->gid); out: close(sockfd); @@ -184,7 +196,8 @@ out: static struct pingpong_dest *pp_server_exch_dest(struct pingpong_context *ctx, int ib_port, int port, int sl, - const struct pingpong_dest *my_dest) + const struct pingpong_dest *my_dest, + int sgid_idx) { struct addrinfo *res, *t; struct addrinfo hints = { @@ -193,10 +206,11 @@ static struct pingpong_dest *pp_server_e .ai_socktype = SOCK_STREAM }; char *service; - char msg[sizeof "0000:000000:000000"]; + char msg[sizeof "0000:000000:000000:00000000000000000000000000000000"]; int n; int sockfd = -1, connfd; struct pingpong_dest *rem_dest = NULL; + char gid[33]; if (asprintf(&service, "%d", port) < 0) return NULL; @@ -250,16 +264,18 @@ static struct pingpong_dest *pp_server_e if (!rem_dest) goto out; - sscanf(msg, "%x:%x:%x", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn); + sscanf(msg, "%x:%x:%x:%s", &rem_dest->lid, &rem_dest->qpn, &rem_dest->psn, gid); + wire_gid_to_gid(gid, &rem_dest->gid); - if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest)) { + if (pp_connect_ctx(ctx, ib_port, my_dest->psn, sl, rem_dest, sgid_idx)) { fprintf(stderr, "Couldn't connect to remote QP\n"); free(rem_dest); rem_dest = NULL; goto out; } - sprintf(msg, "%04x:%06x:%06x", my_dest->lid, my_dest->qpn, my_dest->psn); + gid_to_wire_gid(&my_dest->gid, gid); + sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); if (write(connfd, msg, sizeof msg) != sizeof msg) { fprintf(stderr, "Couldn't send local address\n"); free(rem_dest); @@ -474,10 +490,11 @@ static void usage(const char *argv0) printf(" -p, --port= listen on/connect to port (default 18515)\n"); printf(" -d, --ib-dev= use IB device (default first device found)\n"); printf(" -i, --ib-port= use port of IB device (default 1)\n"); - printf(" -s, --size= size of message to exchange (default 2048)\n"); + printf(" -s, --size= size of message to exchange (default 1024)\n"); printf(" -r, --rx-depth= number of receives to post at a time (default 500)\n"); printf(" -n, --iters= number of exchanges (default 1000)\n"); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***