Date: Mon, 5 Nov 2007 15:28:03 GMT From: Steve Wise <swise@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 128682 for review Message-ID: <200711051528.lA5FS38Z093243@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=128682 Change 128682 by swise@swise:vic10:iwarp on 2007/11/05 15:27:49 Get rdma_add.c to compile and load. Affected files ... .. //depot/projects/iwarp/sys/contrib/rdma/ib_addr.h#3 edit .. //depot/projects/iwarp/sys/contrib/rdma/rdma_addr.c#2 edit Differences ... ==== //depot/projects/iwarp/sys/contrib/rdma/ib_addr.h#3 (text+ko) ==== @@ -31,10 +31,15 @@ #define IB_ADDR_H #include <contrib/rdma/ib_verbs.h> +#include <net/if.h> +#include <net/ethernet.h> + +#define MAX_ADDR_LEN ETHER_ADDR_LEN /* XXX doesn't support IB! */ struct rdma_addr_client { atomic_t refcount; struct cv comp; + struct mtx lock; }; /** @@ -86,7 +91,7 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr); -int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev, +int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev, const unsigned char *dst_dev_addr); static inline int ip_addr_size(struct sockaddr *addr) ==== //depot/projects/iwarp/sys/contrib/rdma/rdma_addr.c#2 (text+ko) ==== @@ -31,9 +31,11 @@ #include <sys/cdefs.h> #include <sys/param.h> +#include <sys/condvar.h> #include <sys/systm.h> #include <sys/kernel.h> #include <sys/socket.h> +#include <sys/module.h> #include <sys/linux_compat.h> #include <sys/lock.h> @@ -41,10 +43,18 @@ #include <sys/mutex.h> #include <sys/rwlock.h> #include <sys/queue.h> +#include <sys/taskqueue.h> + +#include <net/if.h> +#include <net/if_dl.h> +#include <net/if_var.h> +#include <net/if_arp.h> +#include <net/route.h> +#include <net80211/ieee80211_freebsd.h> -#include <net/if_arp.h> #include <netinet/in.h> +#include <netinet/if_ether.h> #include <contrib/rdma/ib_addr.h> @@ -67,50 +77,51 @@ int status; }; -static void process_req(struct work_struct *work); +static void process_req(void *ctx, int pending); static struct mtx lock; -static TAILQ_HEAD(, addr_req) req_list; -static DECLARE_DELAYED_WORK(work, process_req); -static struct taskqueue *addr_wq; +static TAILQ_HEAD(addr_req_list, addr_req) req_list; +static struct task addr_task; +static struct taskqueue *addr_taskq; +static struct callout addr_ch; +static eventhandler_tag route_event_tag; + +static void addr_timeout(void *arg) +{ + taskqueue_enqueue(addr_taskq, &addr_task); +} void rdma_addr_register_client(struct rdma_addr_client *client) { atomic_set(&client->refcount, 1); - init_completion(&client->comp); + mtx_init(&client->lock, "rdma_addr client lock", NULL, MTX_DEF); + cv_init(&client->comp, "rdma_addr cv"); } EXPORT_SYMBOL(rdma_addr_register_client); static inline void put_client(struct rdma_addr_client *client) { + mtx_lock(&client->lock); if (atomic_dec_and_test(&client->refcount)) - complete(&client->comp); + cv_signal(&client->comp); + mtx_unlock(&client->lock); } void rdma_addr_unregister_client(struct rdma_addr_client *client) { put_client(client); - wait_for_completion(&client->comp); + mtx_lock(&client->lock); + cv_wait(&client->comp, &client->lock); } EXPORT_SYMBOL(rdma_addr_unregister_client); -int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev, +int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev, const unsigned char *dst_dev_addr) { - switch (dev->type) { - case ARPHRD_INFINIBAND: - dev_addr->dev_type = RDMA_NODE_IB_CA; - break; - case ARPHRD_ETHER: - dev_addr->dev_type = RDMA_NODE_RNIC; - break; - default: - return (EADDRNOTAVAIL); - } - - memcpy(dev_addr->src_dev_addr, dev->dev_addr, MAX_ADDR_LEN); - memcpy(dev_addr->broadcast, dev->broadcast, MAX_ADDR_LEN); + dev_addr->dev_type = RDMA_NODE_RNIC; + memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), MAX_ADDR_LEN); + memcpy(dev_addr->broadcast, dev->if_broadcastaddr, MAX_ADDR_LEN); if (dst_dev_addr) memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN); return 0; @@ -119,167 +130,138 @@ int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr) { - struct net_device *dev; - __be32 ip = ((struct sockaddr_in *) addr)->sin_addr.s_addr; int ret; + struct route iproute; + struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst; - dev = ip_dev_find(ip); - if (!dev) + bzero(&iproute, sizeof iproute); + dst->sin_family = AF_INET; + dst->sin_len = sizeof dst; + dst->sin_addr.s_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr; + + rtalloc(&iproute); + if (iproute.ro_rt == NULL); return (EADDRNOTAVAIL); - ret = rdma_copy_addr(dev_addr, dev, NULL); - dev_put(dev); + ret = rdma_copy_addr(dev_addr, iproute.ro_rt->rt_ifp, NULL); + rtfree(iproute.ro_rt); return ret; } EXPORT_SYMBOL(rdma_translate_ip); static void queue_req(struct addr_req *req) { - struct addr_req *temp_req; + struct addr_req *tmp_req; mtx_lock(&lock); - TAILQ_FOREACH_REVERSE(tmp_req, &req_list, req_list, entry) - if (time_after_eq(req->timeout, temp_req->timeout)) + TAILQ_FOREACH_REVERSE(tmp_req, &req_list, addr_req_list, entry) + if (time_after_eq(req->timeout, tmp_req->timeout)) break; TAILQ_INSERT_AFTER(&req_list, tmp_req, req, entry); if (TAILQ_FIRST(&req_list) == req) - callout_reset(&work, req->timeout - ticks); + callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, NULL); mtx_unlock(&lock); } static void addr_send_arp(struct sockaddr_in *dst_in) { - struct rtenry *rt; - u32 dst_ip = dst_in->sin_addr.s_addr; + struct route iproute; + struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst; + char dmac[ETHER_ADDR_LEN]; + + bzero(&iproute, sizeof iproute); + *dst = *dst_in; - memset(&fl, 0, sizeof fl); - fl.nl_u.ip4_u.daddr = dst_ip; - if (ip_route_output_key(&rt, &fl)) + rtalloc(&iproute); + if (iproute.ro_rt == NULL); return; - arp_send(ARPOP_REQUEST, ETH_P_ARP, rt->rt_gateway, rt->idev->dev, - rt->rt_src, NULL, rt->idev->dev->dev_addr, NULL); - ip_rt_put(rt); + arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL, + rt_key(iproute.ro_rt), dmac); + + rtfree(iproute.ro_rt); } static int addr_resolve_remote(struct sockaddr_in *src_in, struct sockaddr_in *dst_in, struct rdma_dev_addr *addr) { - u32 src_ip = src_in->sin_addr.s_addr; - u32 dst_ip = dst_in->sin_addr.s_addr; - struct flowi fl; - struct rtable *rt; - struct neighbour *neigh; - int ret; + int ret = 0; + struct route iproute; + struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst; + char dmac[ETHER_ADDR_LEN]; + + bzero(&iproute, sizeof iproute); + *dst = *dst_in; - memset(&fl, 0, sizeof fl); - fl.nl_u.ip4_u.daddr = dst_ip; - fl.nl_u.ip4_u.saddr = src_ip; - ret = ip_route_output_key(&rt, &fl); - if (ret) + rtalloc(&iproute); + if (iproute.ro_rt == NULL); goto out; /* If the device does ARP internally, return 'done' */ - if (rt->idev->dev->flags & IFF_NOARP) { - rdma_copy_addr(addr, rt->idev->dev, NULL); + if (iproute.ro_rt->rt_ifp->if_flags & IFF_NOARP) { + rdma_copy_addr(addr, iproute.ro_rt->rt_ifp, NULL); goto put; } - - neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev); - if (!neigh) { - ret = ENODATA; + ret = arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL, + rt_key(iproute.ro_rt), dmac); + if (ret) { goto put; } - if (!(neigh->nud_state & NUD_VALID)) { - ret = ENODATA; - goto release; - } - - if (!src_ip) { + if (!src_in->sin_addr.s_addr) { + src_in->sin_len = sizeof *src_in; src_in->sin_family = dst_in->sin_family; - src_in->sin_addr.s_addr = rt->rt_src; + src_in->sin_addr.s_addr = ((struct sockaddr_in *)iproute.ro_rt->rt_ifa->ifa_addr)->sin_addr.s_addr; } - ret = rdma_copy_addr(addr, neigh->dev, neigh->ha); -release: - neigh_release(neigh); + ret = rdma_copy_addr(addr, iproute.ro_rt->rt_ifp, dmac); put: - ip_rt_put(rt); + rtfree(iproute.ro_rt); out: return ret; } -static void process_req(struct work_struct *work) +static void process_req(void *ctx, int pending) { - struct addr_req *req, *temp_req; + struct addr_req *req, *tmp_req; struct sockaddr_in *src_in, *dst_in; - struct list_head done_list; + TAILQ_HEAD(, addr_req) done_list; TAILQ_INIT(&done_list); mtx_lock(&lock); - list_for_each_entry_safe(req, temp_req, &req_list, list) { - if (req->status == ENODATA) { + TAILQ_FOREACH_SAFE(req, &req_list, entry, tmp_req) { + if (req->status == EWOULDBLOCK) { src_in = (struct sockaddr_in *) &req->src_addr; dst_in = (struct sockaddr_in *) &req->dst_addr; req->status = addr_resolve_remote(src_in, dst_in, req->addr); - if (req->status && time_after_eq(jiffies, req->timeout)) + if (req->status && time_after_eq(ticks, req->timeout)) req->status = ETIMEDOUT; - else if (req->status == ENODATA) + else if (req->status == EWOULDBLOCK) continue; } - list_move_tail(&req->list, &done_list); + TAILQ_REMOVE(&req_list, req, entry); + TAILQ_INSERT_TAIL(&done_list, req, entry); } - if (!list_empty(&req_list)) { - req = list_entry(req_list.next, struct addr_req, list); - callout_reset(&work, req->timeout - ticks); + if (!TAILQ_EMPTY(&req_list)) { + req = TAILQ_FIRST(&req_list); + callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, + NULL); } mtx_unlock(&lock); - list_for_each_entry_safe(req, temp_req, &done_list, list) { - list_del(&req->list); + TAILQ_FOREACH_SAFE(req, &done_list, entry, tmp_req) { + TAILQ_REMOVE(&done_list, req, entry); req->callback(req->status, &req->src_addr, req->addr, req->context); put_client(req->client); - kfree(req); - } -} - -static int addr_resolve_local(struct sockaddr_in *src_in, - struct sockaddr_in *dst_in, - struct rdma_dev_addr *addr) -{ - struct net_device *dev; - u32 src_ip = src_in->sin_addr.s_addr; - __be32 dst_ip = dst_in->sin_addr.s_addr; - int ret; - - dev = ip_dev_find(dst_ip); - if (!dev) - return (EADDRNOTAVAIL); - - if (ZERONET(src_ip)) { - src_in->sin_family = dst_in->sin_family; - src_in->sin_addr.s_addr = dst_ip; - ret = rdma_copy_addr(addr, dev, dev->dev_addr); - } else if (LOOPBACK(src_ip)) { - ret = rdma_translate_ip((struct sockaddr *)dst_in, addr); - if (!ret) - memcpy(addr->dst_dev_addr, dev->dev_addr, MAX_ADDR_LEN); - } else { - ret = rdma_translate_ip((struct sockaddr *)src_in, addr); - if (!ret) - memcpy(addr->dst_dev_addr, dev->dev_addr, MAX_ADDR_LEN); + free(req, M_DEVBUF); } - - dev_put(dev); - return ret; } int rdma_resolve_ip(struct rdma_addr_client *client, @@ -293,7 +275,7 @@ struct addr_req *req; int ret = 0; - req = kmalloc(sizeof *req, GFP_KERNEL); + req = malloc(sizeof *req, M_DEVBUF, M_WAITOK); if (!req) return (ENOMEM); memset(req, 0, sizeof *req); @@ -310,24 +292,22 @@ src_in = (struct sockaddr_in *) &req->src_addr; dst_in = (struct sockaddr_in *) &req->dst_addr; - req->status = addr_resolve_local(src_in, dst_in, addr); - if (req->status == EADDRNOTAVAIL) - req->status = addr_resolve_remote(src_in, dst_in, addr); + req->status = addr_resolve_remote(src_in, dst_in, addr); switch (req->status) { case 0: - req->timeout = jiffies; + req->timeout = ticks; queue_req(req); break; - case ENODATA: - req->timeout = msecs_to_jiffies(timeout_ms) + jiffies; + case EWOULDBLOCK: + req->timeout = msecs_to_ticks(timeout_ms) + ticks; queue_req(req); addr_send_arp(dst_in); break; default: ret = req->status; atomic_dec(&client->refcount); - kfree(req); + free(req, M_DEVBUF); break; } return ret; @@ -336,15 +316,16 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr) { - struct addr_req *req, *temp_req; + struct addr_req *req, *tmp_req; mtx_lock(&lock); - list_for_each_entry_safe(req, temp_req, &req_list, list) { + TAILQ_FOREACH_SAFE(req, &req_list, entry, tmp_req) { if (req->addr == addr) { req->status = ECANCELED; - req->timeout = jiffies; - list_move(&req->list, &req_list); - callout_reset(&work, req->timeout - ticks); + req->timeout = ticks; + TAILQ_REMOVE(&req_list, req, entry); + TAILQ_INSERT_HEAD(&req_list, req, entry); + callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, NULL); break; } } @@ -352,36 +333,77 @@ } EXPORT_SYMBOL(rdma_addr_cancel); -static int netevent_callback(struct notifier_block *self, unsigned long event, - void *ctx) +static void +route_event_callback(void *unused, int event, struct rtentry *rt0, struct rtentry *rt1) { - if (event == NETEVENT_NEIGH_UPDATE) { - struct neighbour *neigh = ctx; - - if (neigh->nud_state & NUD_VALID) { - callout_reset(&work, 0); - } + if (event == RTEVENT_ARP_UPDATE) { + callout_stop(&addr_ch); + taskqueue_enqueue(addr_taskq, &addr_task); } - return 0; } -static struct notifier_block nb = { - .notifier_call = netevent_callback -}; - static int addr_init(void) { + TAILQ_INIT(&req_list); + mtx_init(&lock, "rdma_addr req_list lock", NULL, MTX_DEF); + + addr_taskq = taskqueue_create("rdma_addr_taskq", M_WAITOK, + taskqueue_thread_enqueue, &addr_taskq); + if (addr_taskq == NULL) { + printf("failed to allocate rdma_addr taskqueue\n"); + return (ENOMEM); + } + taskqueue_start_threads(&addr_taskq, 1, PI_NET, "rdma_addr taskq"); + TASK_INIT(&addr_task, 0, process_req, NULL); + callout_init(&addr_ch, TRUE); - register_netevent_notifier(&nb); + route_event_tag = EVENTHANDLER_REGISTER(route_event, + route_event_callback, NULL, EVENTHANDLER_PRI_ANY); + return 0; } static void addr_cleanup(void) { - unregister_netevent_notifier(&nb); - taskqueue_free(addr_wq); + EVENTHANDLER_DEREGISTER(route_event_callback, route_event_tag); + callout_stop(&addr_ch); + taskqueue_drain(addr_taskq, &addr_task); + taskqueue_free(addr_taskq); +} + +static int +addr_load(module_t mod, int cmd, void *arg) +{ + int err = 0; + + switch (cmd) { + case MOD_LOAD: + printf("Loading rdma_addr.\n"); + + addr_init(); + break; + case MOD_QUIESCE: + break; + case MOD_UNLOAD: + printf("Unloading rdma_addr.\n"); + addr_cleanup(); + break; + case MOD_SHUTDOWN: + break; + default: + err = EOPNOTSUPP; + break; + } + + return (err); } -module_init(addr_init); -module_exit(addr_cleanup); +static moduledata_t mod_data = { + "rdma_addr", + addr_load, + 0 +}; + +MODULE_VERSION(rdma_addr, 1); +DECLARE_MODULE(rdma_addr, mod_data, SI_SUB_EXEC, SI_ORDER_ANY);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200711051528.lA5FS38Z093243>