Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 04 Jan 2004 03:43:01 +0900
From:      takawata@jp.freebsd.org
To:        hackers@freebsd.org
Subject:   ng_ksocket ACCEPT lock related problem.
Message-ID:  <200401031843.i03Ih1wa082382@sana.init-main.com>

next in thread | raw e-mail | index | archive | help
Hi, I noticed ng_ksocket can listen for stream type connection
and wrote a test program like this, and it works as I expected,
but it also produced LoR problem etc.  Is this known problem?

===Message
malloc() of "16" with the following non-sleepable locks held:
exclusive sleep mutex inp (tcpinp) r = 0 (0xc458aa68) locked @ netinet/tcp_input.c:653
exclusive sleep mutex tcp r = 0 (0xc067186c) locked @ netinet/tcp_usrreq.c:480
lock order reversal
 1st 0xc458a84c inp (tcpinp) @ netinet/tcp_input.c:653
 2nd 0xc067186c tcp (tcp) @ netinet/tcp_usrreq.c:621
Stack backtrace:
backtrace(c0600143,c067186c,c0605c04,c0605c04,c06072d9) at backtrace+0x17
witness_lock(c067186c,8,c06072d9,26d,0) at witness_lock+0x672
_mtx_lock_flags(c067186c,0,c06072d0,26d,0) at _mtx_lock_flags+0xba
tcp_usr_rcvd(c47f8b40,80,c05fc199,14b,3b9aca00) at tcp_usr_rcvd+0x30
soreceive(c47f8b40,0,d75bd9e4,d75bd9dc,0) at soreceive+0x8ef
ng_ksocket_incoming2(c486ab00,0,c47f8b40,4,c475d458) at ng_ksocket_incoming2+0x2
4f
ng_apply_item(c486ab00,c475d440,c48811ee,7d6,d75bda98) at ng_apply_item+0x4bf
ng_snd_item(c475d440,0,c04ce15c,c47f8b8c,c47f8b40) at ng_snd_item+0x779
ng_send_fn(c486ab00,0,c488aad0,c47f8b40,4) at ng_send_fn+0x1b7
ng_ksocket_incoming(c47f8b40,c486ab00,4,34,11800) at ng_ksocket_incoming+0x2f
sowakeup(c47f8b40,c47f8b8c,c0606902,444,5000) at sowakeup+0xa1
tcp_input(c1934400,14,c064b620,c0671474,246) at tcp_input+0x1327
ip_input(c1934400,0,c0605432,95,c0670d38) at ip_input+0x8e3
netisr_processqueue(c0670d38,0,c0605432,fd,c1915680) at netisr_processqueue+0x8e

swi_net(0,0,c05faa43,23a,c191c54c) at swi_net+0xa3
ithread_loop(c4367380,d75bdd48,c05fa8b1,311,0) at ithread_loop+0x192
fork_exit(c048e580,c4367380,d75bdd48) at fork_exit+0xb4
fork_trampoline() at fork_trampoline+0x8
--- trap 0x1, eip = 0, esp = 0xd75bdd7c, ebp = 0 ---
==

=======Sample server
#include <sys/types.h>
#include <sys/event.h>
#include <sys/time.h>
#include <stdio.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netdb.h>
#include <netgraph.h>
#include <netgraph/ng_message.h>
#include <netgraph/ng_ksocket.h>
struct {
  int cs, ds;
}allinfo;
void in_read_handler(struct kevent *kev)
{
  int len;
  char buf[8192];
  len = read(0, buf, sizeof(buf));
  NgSendData(allinfo.ds, "slave0", buf, len);
}
void c_read_handler(struct kevent *kev)
{
  struct ng_mesg *rep;
  struct ng_ksocket_accept *acp;
  u_char buffer[sizeof(struct ng_mesg)+ sizeof(u_int32_t) + SOCK_MAXADDRLEN];
  char path[NG_PATHSIZ];
  struct ngm_connect conn;

  rep = (struct ng_mesg *) buffer;
  if(NgRecvMsg(allinfo.cs, rep, sizeof(buffer) , path) < 0 ){
    perror("RecvMsg");
    return ;
  }
  printf("Path:%s\n", path);
  printf("GENERIC: %x %d %d %x  %x %x %s \n",
	 rep->header.version, rep->header.arglen, 
	 rep->header.flags, rep->header.token, rep->header.typecookie,
	 rep->header.cmd, rep->header.cmdstr);
  if((rep->header.typecookie == NGM_KSOCKET_COOKIE)&&
     (rep->header.cmd == NGM_KSOCKET_ACCEPT)){
    acp = (struct ng_ksocket_accept *) rep->data;
    printf("ACCEPT: nodeid %x\n", acp->nodeid);
    strcpy(conn.path, path);
    sprintf(conn.ourhook, "slave%d", 0);
    sprintf(conn.peerhook, "inet/stream/tcp", 0);
    if(NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_CONNECT,
		 &conn, sizeof(conn)) < 0){
      perror("SendMsg");
      return;
    }
  }
}
char notfound_msg[]="HTTP/1.1 404  NOT Found\n\n<html><title>Not Found</title><body> Not Found</body></html>";

void d_read_handler(struct kevent *kev)
{
  char hook[NG_HOOKSIZ];
  char buf[8192];
  int len;
  len = NgRecvData(allinfo.ds, buf, sizeof(buf), hook);
  printf("%d Data from %s\n", len,  hook);
  if(len == 0){
    printf("HEHE\n");
    if(NgSendMsg(allinfo.cs, ".:", NGM_GENERIC_COOKIE, NGM_RMHOOK, hook, sizeof(struct ngm_rmhook)) < 0){
      perror("RMHOOK"); 
      return ;
    }
    if((NgSendMsg(allinfo.cs, "master0",
		  NGM_KSOCKET_COOKIE, NGM_KSOCKET_ACCEPT,
		  NULL, 0)) < 0){
      perror("ACCEPT");
    }
  }

}

void d_write_handler(struct kevent *kev)
{
}

#define MAXID 16
typedef  void (*kevhandler)(struct kevent *);

int main(int argc, char * argv[])
{
  int id[MAXID];
  int kq, error;
  struct addrinfo  hints, *res, *res0;
  int i,inqueue,numlisten = 1;
  struct kevent ev[4];
  kevhandler evh;
  struct ngm_mkpeer ngmkpeer;
  struct sockaddr sa;

  
  if((kq = kqueue()) < 0){
    perror("kqueue");
    exit(-1);
  }

  if(NgMkSockNode("test", &allinfo.cs, &allinfo.ds) == -1){
    perror("NgMkSockNode");
  }
  

  EV_SET(&ev[0], allinfo.cs, EVFILT_READ, EV_ADD, 0, 0, c_read_handler);
  EV_SET(&ev[1], allinfo.ds, EVFILT_READ, EV_ADD, 0, 0, d_read_handler);
  EV_SET(&ev[2], 0, EVFILT_READ, EV_ADD, 0, 0, in_read_handler);
  kevent(kq, ev, 3, NULL, 0, NULL);

  memset(&hints, 0, sizeof(hints));

  hints.ai_family = PF_INET;
  hints.ai_socktype = SOCK_STREAM;
  if((error = getaddrinfo(NULL, "http", &hints, &res0))){
    errx(1, "%s", gai_strerror(error));
  }

  for(res = res0,i = 0;  res && i < MAXID ; res = res->ai_next, i++){
    snprintf(ngmkpeer.type, sizeof(ngmkpeer.type), "%s", "ksocket");
    snprintf(ngmkpeer.ourhook, sizeof(ngmkpeer.ourhook), "master%d", i);
    snprintf(ngmkpeer.peerhook, sizeof(ngmkpeer.peerhook), 
	     "%d/%d/%d", res->ai_family, res->ai_socktype, res->ai_protocol);
    if((id[i] = NgSendMsg(allinfo.cs, ".:",
		       NGM_GENERIC_COOKIE, NGM_MKPEER,
		       &ngmkpeer, sizeof(ngmkpeer))) < 0){
      
      perror("NgSendMsg");
      continue;
    }
    
    if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook,
		       NGM_KSOCKET_COOKIE, NGM_KSOCKET_BIND,
		       res->ai_addr, 
		       res->ai_addrlen)) < 0){
      
      perror("NgSendMsg");
      exit(-1);
    }

    if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook,
		  NGM_KSOCKET_COOKIE, NGM_KSOCKET_LISTEN,
		  &numlisten,
		  sizeof(numlisten))) < 0){
      
      perror("NgSendMsg");
      exit(-1);
    }

  }
  for(i = 0 ; i < numlisten; i++){
    if((NgSendMsg(allinfo.cs, ngmkpeer.ourhook,
		  NGM_KSOCKET_COOKIE, NGM_KSOCKET_ACCEPT,
		  NULL,
		  0)) < 0){
      perror("NgSendMsg");
      exit(-1);
    }
  }    
  while(1){
    inqueue = kevent(kq, NULL, 0, ev, 4, NULL);
    printf("Inqueue %d\n", inqueue);
    for(i=0; i < inqueue; i++){
      evh = ev[i].udata;
      if(evh!=NULL){
	printf("%x\n", evh);
	(evh)(&ev[i]);
      }
    }
    printf("\n");
  }
}
======



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200401031843.i03Ih1wa082382>