Skip site navigation (1)Skip section navigation (2) 
Date:      Tue, 23 Jul 2013 00:09:31 -0700
From:      trafdev <trafdev@mail.ru>
To:        Adrian Chadd <adrian@freebsd.org>
Cc:        Sepherosa Ziehau <sepherosa@gmail.com>, freebsd-net@freebsd.org, John-Mark Gurney <jmg@funkthat.com>
Subject:   Re: SO_REUSEPORT: strange kernel balancer behaviour
Message-ID:  <51EE2C2B.4020800@mail.ru>
In-Reply-To: <CAJ-Vmons_6XfywPm2SPnE1-aac8Lc-wBEN06kiezD2m49z=PYw@mail.gmail.com>
References:  <51E0E2AF.7090404@mail.ru> <CAMOc5cz6gP2N62T4QhbTdVar94O4FSdPDsqktD_9vJ0mYVqt_Q@mail.gmail.com> <51E44E2F.8060700@mail.ru> <CAJ-VmomHHfhExa4g63tT_sf0hTPa2T7jPKQGHrD0fchq=-k%2B=g@mail.gmail.com> <51E455D5.2090403@mail.ru> <20130722200205.GO26412@funkthat.com> <51EDA37A.9040200@mail.ru> <CAJ-Vmon-WdsUnH33smkf%2B33yrUHP0p-Fp3ng1ZyTsvLb9tQfVA@mail.gmail.com> <51EE198B.7040509@mail.ru> <CAJ-Vmokxo1LniNvKYerap8bRe9hpUWLvdzSYquNVyZX4Ata_sQ@mail.gmail.com> <CAJ-Vmons_6XfywPm2SPnE1-aac8Lc-wBEN06kiezD2m49z=PYw@mail.gmail.com>
next in thread | previous in thread | raw e-mail | index | archive | help 
It's like shared acceptor FD and N processes:

Listening proc:

bool HttpServer::Listen(unsigned short port, uint listen_backlog) {
    LOG4CXX_TRACE(kLogger, "Listen");
    if ((sockd_acceptor_ = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "socket");
        return false;
    }
    struct sockaddr_in sa_in;
    memset(&sa_in, 0, sizeof(sa_in));
    sa_in.sin_family = AF_INET;
    sa_in.sin_port = htons(port);
    sa_in.sin_addr.s_addr = htonl(INADDR_ANY);
    int yes = 1;
    if (setsockopt(sockd_acceptor_, SOL_SOCKET, SO_REUSEPORT, &yes, 
sizeof (yes)) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "setsockopt");
        return false;
    }
    if (bind(sockd_acceptor_, (const struct sockaddr*)&sa_in, 
sizeof(sa_in)) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "bind");
        return false;
    }
    if (listen(sockd_acceptor_, listen_backlog) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "socket listen");
        return false;
    }
    if (!fcntl_set(sockd_acceptor_, O_NONBLOCK))
        return false;

    sockd_acceptor_watcher_.set<HttpServer, &HttpServer::AcceptConnCb> 
(this);
    sockd_acceptor_watcher_.start(sockd_acceptor_, ev::READ);

    int wrk = thread::hardware_concurrency();
    while (--wrk) {
        pid_t pid = fork();
        if (pid == 0) {
            LOG4CXX_INFO(kLogger, "process " << wrk <<  " started");
            ev::default_loop().post_fork();
            ev::default_loop().run();
            return true;
        }
    }

    ev::default_loop().run();
    return true;
}


Accept conn callback:

void HttpServer::AcceptConnCb(ev::io &w, int revents) {
    LOG4CXX_TRACE(kLogger, "AcceptConnCb");
    CHECK_LIBEV_ERROR
    struct sockaddr_in addr;
    socklen_t addr_len = sizeof (addr);
    int sockd;
    if ((sockd = accept(sockd_acceptor_, (struct sockaddr *) &addr,
            &addr_len)) == -1) {
        if (errno == EAGAIN || errno == EWOULDBLOCK) {
            return;
        } else {
            LOG4CXX_ERROR_ERRNO(kLogger, "accept");
            return;
        }
    }
    if (!fcntl_set(sockd, O_NONBLOCK))
        return;
    char str[INET_ADDRSTRLEN];
    if (!inet_ntop(AF_INET, &(addr.sin_addr), str, INET_ADDRSTRLEN)) {
        LOG4CXX_ERROR(kLogger, "inet_ntop");
        return;
    }
    // Internal logic follows... ConnIn* c = 
ConnPool::Instance().Add(sockd, str);
    // c->Recv(this);
    return;
}

Accept conn callback is called in N processes on each connection, only 
one wins,
others exit by errno == EAGAIN case. Overhead is almost zero.
Problem is that "wins" distribution is far from equal.


On Mon Jul 22 23:37:57 2013, Adrian Chadd wrote:
> eg:
>
> * one process, one listen thread, multiple dispatch threads?
> * n processes, one listen FD per process, all listening on the same IP:port?
> * one process, each thread listening on the same IP:port?
> * something else?
>
> Thanks,
>
>
>
> -adrian
>
>

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?51EE2C2B.4020800>