From owner-freebsd-hackers@FreeBSD.ORG Sat Jan 3 10:46:42 2004 Return-Path: Delivered-To: freebsd-hackers@freebsd.org Received: from mx1.FreeBSD.org (mx1.freebsd.org [216.136.204.125]) by hub.freebsd.org (Postfix) with ESMTP id 9EF0A16A4CE for ; Sat, 3 Jan 2004 10:46:42 -0800 (PST) Received: from sana.init-main.com (104.194.138.210.bn.2iij.net [210.138.194.104]) by mx1.FreeBSD.org (Postfix) with ESMTP id 7BA3943D58 for ; Sat, 3 Jan 2004 10:46:36 -0800 (PST) (envelope-from takawata@init-main.com) Received: from init-main.com (localhost [127.0.0.1]) by sana.init-main.com (8.12.10/8.12.9) with ESMTP id i03Ih1wa082382 for ; Sun, 4 Jan 2004 03:43:02 +0900 (JST) (envelope-from takawata@init-main.com) Message-Id: <200401031843.i03Ih1wa082382@sana.init-main.com> From: takawata@jp.freebsd.org To: hackers@freebsd.org Date: Sun, 04 Jan 2004 03:43:01 +0900 Sender: takawata@init-main.com Subject: ng_ksocket ACCEPT lock related problem. X-BeenThere: freebsd-hackers@freebsd.org X-Mailman-Version: 2.1.1 Precedence: list List-Id: Technical Discussions relating to FreeBSD List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 03 Jan 2004 18:46:42 -0000 Hi, I noticed ng_ksocket can listen for stream type connection and wrote a test program like this, and it works as I expected, but it also produced LoR problem etc. Is this known problem? ===Message malloc() of "16" with the following non-sleepable locks held: exclusive sleep mutex inp (tcpinp) r = 0 (0xc458aa68) locked @ netinet/tcp_input.c:653 exclusive sleep mutex tcp r = 0 (0xc067186c) locked @ netinet/tcp_usrreq.c:480 lock order reversal 1st 0xc458a84c inp (tcpinp) @ netinet/tcp_input.c:653 2nd 0xc067186c tcp (tcp) @ netinet/tcp_usrreq.c:621 Stack backtrace: backtrace(c0600143,c067186c,c0605c04,c0605c04,c06072d9) at backtrace+0x17 witness_lock(c067186c,8,c06072d9,26d,0) at witness_lock+0x672 _mtx_lock_flags(c067186c,0,c06072d0,26d,0) at _mtx_lock_flags+0xba tcp_usr_rcvd(c47f8b40,80,c05fc199,14b,3b9aca00) at tcp_usr_rcvd+0x30 soreceive(c47f8b40,0,d75bd9e4,d75bd9dc,0) at soreceive+0x8ef ng_ksocket_incoming2(c486ab00,0,c47f8b40,4,c475d458) at ng_ksocket_incoming2+0x2 4f ng_apply_item(c486ab00,c475d440,c48811ee,7d6,d75bda98) at ng_apply_item+0x4bf ng_snd_item(c475d440,0,c04ce15c,c47f8b8c,c47f8b40) at ng_snd_item+0x779 ng_send_fn(c486ab00,0,c488aad0,c47f8b40,4) at ng_send_fn+0x1b7 ng_ksocket_incoming(c47f8b40,c486ab00,4,34,11800) at ng_ksocket_incoming+0x2f sowakeup(c47f8b40,c47f8b8c,c0606902,444,5000) at sowakeup+0xa1 tcp_input(c1934400,14,c064b620,c0671474,246) at tcp_input+0x1327 ip_input(c1934400,0,c0605432,95,c0670d38) at ip_input+0x8e3 netisr_processqueue(c0670d38,0,c0605432,fd,c1915680) at netisr_processqueue+0x8e swi_net(0,0,c05faa43,23a,c191c54c) at swi_net+0xa3 ithread_loop(c4367380,d75bdd48,c05fa8b1,311,0) at ithread_loop+0x192 fork_exit(c048e580,c4367380,d75bdd48) at fork_exit+0xb4 fork_trampoline() at fork_trampoline+0x8 --- trap 0x1, eip = 0, esp = 0xd75bdd7c, ebp = 0 --- == =======Sample server #include #include #include #include #include #include #include #include #include #include #include #include #include struct { int cs, ds; }allinfo; void in_read_handler(struct kevent *kev) { int len; char buf[8192]; len = read(0, buf, sizeof(buf)); NgSendData(allinfo.ds, "slave0", buf, len); } void c_read_handler(struct kevent *kev) { struct ng_mesg *rep; struct ng_ksocket_accept *acp; u_char buffer[sizeof(struct ng_mesg)+ sizeof(u_int32_t) + SOCK_MAXADDRLEN]; char path[NG_PATHSIZ]; struct ngm_connect conn; rep = (struct ng_mesg *) buffer; if(NgRecvMsg(allinfo.cs, rep, sizeof(buffer) , path) < 0 ){ perror("RecvMsg"); return ; } printf("Path:%s\n", path); printf("GENERIC: %x %d %d %x %x %x %s \n", rep->header.version, rep->header.arglen, rep->header.flags, rep->header.token, rep->header.typecookie, rep->header.cmd, rep->header.cmdstr); if((rep->header.typecookie == NGM_KSOCKET_COOKIE)&& (rep->header.cmd == NGM_KSOCKET_ACCEPT)){ acp = (struct ng_ksocket_accept *) rep->data; printf("ACCEPT: nodeid %x\n", acp->nodeid); strcpy(conn.path, path); sprintf(conn.ourhook, "slave%d", 0); sprintf(conn.peerhook, "inet/stream/tcp", 0); if(NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_CONNECT, &conn, sizeof(conn)) < 0){ perror("SendMsg"); return; } } } char notfound_msg[]="HTTP/1.1 404 NOT Found\n\nNot Found Not Found"; void d_read_handler(struct kevent *kev) { char hook[NG_HOOKSIZ]; char buf[8192]; int len; len = NgRecvData(allinfo.ds, buf, sizeof(buf), hook); printf("%d Data from %s\n", len, hook); if(len == 0){ printf("HEHE\n"); if(NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_RMHOOK, hook, sizeof(struct ngm_rmhook)) < 0){ perror("RMHOOK"); return ; } if((NgSendMsg(allinfo.cs, "master0", NGM_KSOCKET_COOKIE, NGM_KSOCKET_ACCEPT, NULL, 0)) < 0){ perror("ACCEPT"); } } } void d_write_handler(struct kevent *kev) { } #define MAXID 16 typedef void (*kevhandler)(struct kevent *); int main(int argc, char * argv[]) { int id[MAXID]; int kq, error; struct addrinfo hints, *res, *res0; int i,inqueue,numlisten = 1; struct kevent ev[4]; kevhandler evh; struct ngm_mkpeer ngmkpeer; struct sockaddr sa; if((kq = kqueue()) < 0){ perror("kqueue"); exit(-1); } if(NgMkSockNode("test", &allinfo.cs, &allinfo.ds) == -1){ perror("NgMkSockNode"); } EV_SET(&ev[0], allinfo.cs, EVFILT_READ, EV_ADD, 0, 0, c_read_handler); EV_SET(&ev[1], allinfo.ds, EVFILT_READ, EV_ADD, 0, 0, d_read_handler); EV_SET(&ev[2], 0, EVFILT_READ, EV_ADD, 0, 0, in_read_handler); kevent(kq, ev, 3, NULL, 0, NULL); memset(&hints, 0, sizeof(hints)); hints.ai_family = PF_INET; hints.ai_socktype = SOCK_STREAM; if((error = getaddrinfo(NULL, "http", &hints, &res0))){ errx(1, "%s", gai_strerror(error)); } for(res = res0,i = 0; res && i < MAXID ; res = res->ai_next, i++){ snprintf(ngmkpeer.type, sizeof(ngmkpeer.type), "%s", "ksocket"); snprintf(ngmkpeer.ourhook, sizeof(ngmkpeer.ourhook), "master%d", i); snprintf(ngmkpeer.peerhook, sizeof(ngmkpeer.peerhook), "%d/%d/%d", res->ai_family, res->ai_socktype, res->ai_protocol); if((id[i] = NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_MKPEER, &ngmkpeer, sizeof(ngmkpeer))) < 0){ perror("NgSendMsg"); continue; } if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook, NGM_KSOCKET_COOKIE, NGM_KSOCKET_BIND, res->ai_addr, res->ai_addrlen)) < 0){ perror("NgSendMsg"); exit(-1); } if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook, NGM_KSOCKET_COOKIE, NGM_KSOCKET_LISTEN, &numlisten, sizeof(numlisten))) < 0){ perror("NgSendMsg"); exit(-1); } } for(i = 0 ; i < numlisten; i++){ if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook, NGM_KSOCKET_COOKIE, NGM_KSOCKET_ACCEPT, NULL, 0)) < 0){ perror("NgSendMsg"); exit(-1); } } while(1){ inqueue = kevent(kq, NULL, 0, ev, 4, NULL); printf("Inqueue %d\n", inqueue); for(i=0; i < inqueue; i++){ evh = ev[i].udata; if(evh!=NULL){ printf("%x\n", evh); (evh)(&ev[i]); } } printf("\n"); } } ======