Date: Sun, 04 Jan 2004 03:43:01 +0900 From: takawata@jp.freebsd.org To: hackers@freebsd.org Subject: ng_ksocket ACCEPT lock related problem. Message-ID: <200401031843.i03Ih1wa082382@sana.init-main.com>
next in thread | raw e-mail | index | archive | help
Hi, I noticed ng_ksocket can listen for stream type connection
and wrote a test program like this, and it works as I expected,
but it also produced LoR problem etc. Is this known problem?
===Message
malloc() of "16" with the following non-sleepable locks held:
exclusive sleep mutex inp (tcpinp) r = 0 (0xc458aa68) locked @ netinet/tcp_input.c:653
exclusive sleep mutex tcp r = 0 (0xc067186c) locked @ netinet/tcp_usrreq.c:480
lock order reversal
1st 0xc458a84c inp (tcpinp) @ netinet/tcp_input.c:653
2nd 0xc067186c tcp (tcp) @ netinet/tcp_usrreq.c:621
Stack backtrace:
backtrace(c0600143,c067186c,c0605c04,c0605c04,c06072d9) at backtrace+0x17
witness_lock(c067186c,8,c06072d9,26d,0) at witness_lock+0x672
_mtx_lock_flags(c067186c,0,c06072d0,26d,0) at _mtx_lock_flags+0xba
tcp_usr_rcvd(c47f8b40,80,c05fc199,14b,3b9aca00) at tcp_usr_rcvd+0x30
soreceive(c47f8b40,0,d75bd9e4,d75bd9dc,0) at soreceive+0x8ef
ng_ksocket_incoming2(c486ab00,0,c47f8b40,4,c475d458) at ng_ksocket_incoming2+0x2
4f
ng_apply_item(c486ab00,c475d440,c48811ee,7d6,d75bda98) at ng_apply_item+0x4bf
ng_snd_item(c475d440,0,c04ce15c,c47f8b8c,c47f8b40) at ng_snd_item+0x779
ng_send_fn(c486ab00,0,c488aad0,c47f8b40,4) at ng_send_fn+0x1b7
ng_ksocket_incoming(c47f8b40,c486ab00,4,34,11800) at ng_ksocket_incoming+0x2f
sowakeup(c47f8b40,c47f8b8c,c0606902,444,5000) at sowakeup+0xa1
tcp_input(c1934400,14,c064b620,c0671474,246) at tcp_input+0x1327
ip_input(c1934400,0,c0605432,95,c0670d38) at ip_input+0x8e3
netisr_processqueue(c0670d38,0,c0605432,fd,c1915680) at netisr_processqueue+0x8e
swi_net(0,0,c05faa43,23a,c191c54c) at swi_net+0xa3
ithread_loop(c4367380,d75bdd48,c05fa8b1,311,0) at ithread_loop+0x192
fork_exit(c048e580,c4367380,d75bdd48) at fork_exit+0xb4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xd75bdd7c, ebp = 0 ---
==
=======Sample server
#include <sys/types.h>
#include <sys/event.h>
#include <sys/time.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <netgraph.h>
#include <netgraph/ng_message.h>
#include <netgraph/ng_ksocket.h>
struct {
int cs, ds;
}allinfo;
void in_read_handler(struct kevent *kev)
{
int len;
char buf[8192];
len = read(0, buf, sizeof(buf));
NgSendData(allinfo.ds, "slave0", buf, len);
}
void c_read_handler(struct kevent *kev)
{
struct ng_mesg *rep;
struct ng_ksocket_accept *acp;
u_char buffer[sizeof(struct ng_mesg)+ sizeof(u_int32_t) + SOCK_MAXADDRLEN];
char path[NG_PATHSIZ];
struct ngm_connect conn;
rep = (struct ng_mesg *) buffer;
if(NgRecvMsg(allinfo.cs, rep, sizeof(buffer) , path) < 0 ){
perror("RecvMsg");
return ;
}
printf("Path:%s\n", path);
printf("GENERIC: %x %d %d %x %x %x %s \n",
rep->header.version, rep->header.arglen,
rep->header.flags, rep->header.token, rep->header.typecookie,
rep->header.cmd, rep->header.cmdstr);
if((rep->header.typecookie == NGM_KSOCKET_COOKIE)&&
(rep->header.cmd == NGM_KSOCKET_ACCEPT)){
acp = (struct ng_ksocket_accept *) rep->data;
printf("ACCEPT: nodeid %x\n", acp->nodeid);
strcpy(conn.path, path);
sprintf(conn.ourhook, "slave%d", 0);
sprintf(conn.peerhook, "inet/stream/tcp", 0);
if(NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_CONNECT,
&conn, sizeof(conn)) < 0){
perror("SendMsg");
return;
}
}
}
char notfound_msg[]="HTTP/1.1 404 NOT Found\n\n<html><title>Not Found</title><body> Not Found</body></html>";
void d_read_handler(struct kevent *kev)
{
char hook[NG_HOOKSIZ];
char buf[8192];
int len;
len = NgRecvData(allinfo.ds, buf, sizeof(buf), hook);
printf("%d Data from %s\n", len, hook);
if(len == 0){
printf("HEHE\n");
if(NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_RMHOOK, hook, sizeof(struct ngm_rmhook)) < 0){
perror("RMHOOK");
return ;
}
if((NgSendMsg(allinfo.cs, "master0",
NGM_KSOCKET_COOKIE, NGM_KSOCKET_ACCEPT,
NULL, 0)) < 0){
perror("ACCEPT");
}
}
}
void d_write_handler(struct kevent *kev)
{
}
#define MAXID 16
typedef void (*kevhandler)(struct kevent *);
int main(int argc, char * argv[])
{
int id[MAXID];
int kq, error;
struct addrinfo hints, *res, *res0;
int i,inqueue,numlisten = 1;
struct kevent ev[4];
kevhandler evh;
struct ngm_mkpeer ngmkpeer;
struct sockaddr sa;
if((kq = kqueue()) < 0){
perror("kqueue");
exit(-1);
}
if(NgMkSockNode("test", &allinfo.cs, &allinfo.ds) == -1){
perror("NgMkSockNode");
}
EV_SET(&ev[0], allinfo.cs, EVFILT_READ, EV_ADD, 0, 0, c_read_handler);
EV_SET(&ev[1], allinfo.ds, EVFILT_READ, EV_ADD, 0, 0, d_read_handler);
EV_SET(&ev[2], 0, EVFILT_READ, EV_ADD, 0, 0, in_read_handler);
kevent(kq, ev, 3, NULL, 0, NULL);
memset(&hints, 0, sizeof(hints));
hints.ai_family = PF_INET;
hints.ai_socktype = SOCK_STREAM;
if((error = getaddrinfo(NULL, "http", &hints, &res0))){
errx(1, "%s", gai_strerror(error));
}
for(res = res0,i = 0; res && i < MAXID ; res = res->ai_next, i++){
snprintf(ngmkpeer.type, sizeof(ngmkpeer.type), "%s", "ksocket");
snprintf(ngmkpeer.ourhook, sizeof(ngmkpeer.ourhook), "master%d", i);
snprintf(ngmkpeer.peerhook, sizeof(ngmkpeer.peerhook),
"%d/%d/%d", res->ai_family, res->ai_socktype, res->ai_protocol);
if((id[i] = NgSendMsg(allinfo.cs, ".:",
NGM_GENERIC_COOKIE, NGM_MKPEER,
&ngmkpeer, sizeof(ngmkpeer))) < 0){
perror("NgSendMsg");
continue;
}
if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook,
NGM_KSOCKET_COOKIE, NGM_KSOCKET_BIND,
res->ai_addr,
res->ai_addrlen)) < 0){
perror("NgSendMsg");
exit(-1);
}
if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook,
NGM_KSOCKET_COOKIE, NGM_KSOCKET_LISTEN,
&numlisten,
sizeof(numlisten))) < 0){
perror("NgSendMsg");
exit(-1);
}
}
for(i = 0 ; i < numlisten; i++){
if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook,
NGM_KSOCKET_COOKIE, NGM_KSOCKET_ACCEPT,
NULL,
0)) < 0){
perror("NgSendMsg");
exit(-1);
}
}
while(1){
inqueue = kevent(kq, NULL, 0, ev, 4, NULL);
printf("Inqueue %d\n", inqueue);
for(i=0; i < inqueue; i++){
evh = ev[i].udata;
if(evh!=NULL){
printf("%x\n", evh);
(evh)(&ev[i]);
}
}
printf("\n");
}
}
======
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200401031843.i03Ih1wa082382>