Date: Thu, 19 Mar 2009 05:19:13 GMT From: Yoshihiro Ota <ota@j.email.ne.jp> To: freebsd-gnats-submit@FreeBSD.org Subject: bin/132798: ggated/ggatec Message-ID: <200903190519.n2J5JDGM099384@www.freebsd.org> Resent-Message-ID: <200903190520.n2J5K1IX076982@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 132798
>Category: bin
>Synopsis: ggated/ggatec
>Confidential: no
>Severity: critical
>Priority: low
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: sw-bug
>Submitter-Id: current-users
>Arrival-Date: Thu Mar 19 05:20:00 UTC 2009
>Closed-Date:
>Last-Modified:
>Originator: Yoshihiro Ota
>Release: Since 6.1-RELEASE to 8-CURRENT
>Organization:
>Environment:
%uname -a
FreeBSD xxx 7.1-RELEASE-p3 FreeBSD 7.1-RELEASE-p3 #463 r188634M: Thu Mar 12 12:27:06 EDT 2009 xxx:/usr/obj/usr/src/sys/GENERIC i386
>Description:
Connection between ggate server and client slows down significantly while coying a file, about more than 20MB file size can cause the problem. During slowdown, both server and client wants for each other and its transmission rate is about 1 packet per second. As a result, the system really hangs up on any responses as FS does not response in timely manner. All the commands then takes a couple of minutes easily to response to the user.
Copying 20MB file taking more than 20 minuets is critically wrong.
>How-To-Repeat:
server# echo "192.168.0.0/8 RW /dev/ad1" > /etc/gg.exports
server# ggated
client# ggatec create -orw server /dev/ad1
client# mount /dev/ad1 /mnt/backup
client# rsync -a --delete /home /mnt/backup
This only happens when server CPU is 10 to 20 times slower than client and client writes to the device; as far as NIC, they are the same 100BASE-TX on these hosts. It does not happen with all files, neither.
>Fix:
The problem is ggated and ggatec create 2 TCP uni-directional connections. That is screwing up the TCP layer. Using 1 TCP connection for bi-directional messaging fixes the problem.
Patch attached with submission follows:
diff -r 2fccee0290c5 -r bc62f8d4a1f0 ggatec/ggatec.c
--- a/ggatec/ggatec.c Tue Mar 17 14:17:39 2009 -0400
+++ b/ggatec/ggatec.c Wed Mar 18 14:46:02 2009 -0400
@@ -71,7 +71,7 @@
static off_t mediasize;
static unsigned sectorsize = 0;
static unsigned timeout = G_GATE_TIMEOUT;
-static int sendfd, recvfd;
+static int fd = -1;
static uint32_t token;
static pthread_t sendtd, recvtd;
static int reconnect;
@@ -183,7 +183,7 @@
hdr.gh_error = 0;
g_gate_swap2n_hdr(&hdr);
- data = g_gate_send(sendfd, &hdr, sizeof(hdr), MSG_NOSIGNAL);
+ data = g_gate_send(fd, &hdr, sizeof(hdr), MSG_NOSIGNAL);
g_gate_swap2h_hdr(&hdr);
g_gate_log(LOG_DEBUG, "Sent hdr packet (seq=%llu).", hdr.gh_seq);
send_seq = hdr.gh_seq;
@@ -198,7 +198,7 @@
}
if (hdr.gh_cmd == GGATE_CMD_WRITE) {
- data = g_gate_send(sendfd, ggio.gctl_data,
+ data = g_gate_send(fd, ggio.gctl_data,
ggio.gctl_length, MSG_NOSIGNAL);
if (reconnect)
break;
@@ -231,7 +231,7 @@
ggio.gctl_data = buf;
for (;;) {
- data = g_gate_recv(recvfd, &hdr, sizeof(hdr), MSG_WAITALL);
+ data = g_gate_recv(fd, &hdr, sizeof(hdr), MSG_WAITALL);
if (reconnect)
break;
g_gate_swap2h_hdr(&hdr);
@@ -254,7 +254,7 @@
if (ggio.gctl_error == 0 && ggio.gctl_cmd == GGATE_CMD_READ) {
g_gate_log(LOG_DEBUG, "Received READ packet.");
- data = g_gate_recv(recvfd, ggio.gctl_data,
+ data = g_gate_recv(fd, ggio.gctl_data,
ggio.gctl_length, MSG_WAITALL);
if (reconnect)
break;
@@ -277,7 +277,7 @@
}
static int
-handshake(int dir)
+handshake(void)
{
struct g_gate_version ver;
struct g_gate_cinit cinit;
@@ -367,7 +367,7 @@
close(sfd);
return (-1);
}
- cinit.gc_flags = flags | dir;
+ cinit.gc_flags = flags;
cinit.gc_token = token;
cinit.gc_nconn = 2;
g_gate_swap2n_cinit(&cinit);
@@ -426,15 +426,8 @@
* Our receive descriptor is connected to the send descriptor on the
* server side.
*/
- recvfd = handshake(GGATE_FLAG_SEND);
- if (recvfd == -1)
- return (0);
- /*
- * Our send descriptor is connected to the receive descriptor on the
- * server side.
- */
- sendfd = handshake(GGATE_FLAG_RECV);
- if (sendfd == -1)
+ fd = handshake();
+ if (fd == -1)
return (0);
return (1);
}
@@ -454,8 +447,7 @@
sendtd = pthread_self();
send_thread(NULL);
/* Disconnected. */
- close(sendfd);
- close(recvfd);
+ close(fd);
}
static void
diff -r 2fccee0290c5 -r bc62f8d4a1f0 ggated/ggated.c
--- a/ggated/ggated.c Tue Mar 17 14:17:39 2009 -0400
+++ b/ggated/ggated.c Wed Mar 18 14:46:02 2009 -0400
@@ -65,8 +65,7 @@
unsigned c_sectorsize;
unsigned c_flags; /* flags (RO/RW) */
int c_diskfd;
- int c_sendfd;
- int c_recvfd;
+ int c_netfd;
time_t c_birthtime;
char *c_path;
uint64_t c_token;
@@ -138,10 +137,12 @@
/* Use snprintf(3) so that we don't reenter stdio(3). */
LIST_FOREACH_SAFE(conn, &connections, c_next, tconn) {
snprintf(buf, sizeof(buf),
+ "token %llu\n"
"path %s\n"
"send_seq %llu\n"
"recv_seq %llu\n"
"",
+ conn->c_token,
conn->c_path,
conn->c_send_seq, conn->c_recv_seq);
write(STDERR_FILENO, buf, strlen(buf));
@@ -527,24 +528,11 @@
ip2str((struct sockaddr*)&conn->c_srcaddr),
conn->c_path);
close(conn->c_diskfd);
- close(conn->c_sendfd);
- close(conn->c_recvfd);
+ close(conn->c_netfd);
free(conn->c_path);
free(conn);
}
}
-}
-
-static struct ggd_connection *
-connection_find(struct g_gate_cinit *cinit)
-{
- struct ggd_connection *conn;
-
- LIST_FOREACH(conn, &connections, c_next) {
- if (conn->c_token == cinit->gc_token)
- break;
- }
- return (conn);
}
static struct ggd_connection *
@@ -570,11 +558,8 @@
}
conn->c_token = cinit->gc_token;
memcpy(&conn->c_srcaddr, s, s->sa_len);
- conn->c_sendfd = conn->c_recvfd = -1;
- if ((cinit->gc_flags & GGATE_FLAG_SEND) != 0)
- conn->c_sendfd = sfd;
- else
- conn->c_recvfd = sfd;
+ conn->c_diskfd = -1;
+ conn->c_netfd = sfd;
conn->c_mediasize = 0;
conn->c_sectorsize = 0;
time(&conn->c_birthtime);
@@ -583,32 +568,6 @@
g_gate_log(LOG_DEBUG, "Connection created [%s, %s].", ip2str(s),
conn->c_path);
return (conn);
-}
-
-static int
-connection_add(struct ggd_connection *conn, struct g_gate_cinit *cinit,
- struct sockaddr *s, int sfd)
-{
- if ((cinit->gc_flags & GGATE_FLAG_SEND) != 0) {
- if (conn->c_sendfd != -1) {
- g_gate_log(LOG_WARNING,
- "Send socket already exists [%s, %s].", ip2str(s),
- conn->c_path);
- return (EEXIST);
- }
- conn->c_sendfd = sfd;
- } else {
- if (conn->c_recvfd != -1) {
- g_gate_log(LOG_WARNING,
- "Receive socket already exists [%s, %s].",
- ip2str(s), conn->c_path);
- return (EEXIST);
- }
- conn->c_recvfd = sfd;
- }
- g_gate_log(LOG_DEBUG, "Connection added [%s, %s].", ip2str(s),
- conn->c_path);
- return (0);
}
/*
@@ -622,20 +581,11 @@
LIST_REMOVE(conn, c_next);
g_gate_log(LOG_DEBUG, "Connection removed [%s %s].",
ip2str((struct sockaddr*)&conn->c_srcaddr), conn->c_path);
- if (conn->c_sendfd != -1)
- close(conn->c_sendfd);
- if (conn->c_recvfd != -1)
- close(conn->c_recvfd);
- close(conn->c_diskfd);
+ close(conn->c_netfd);
+ if(conn->c_diskfd == -1)
+ close(conn->c_diskfd);
free(conn->c_path);
free(conn);
-}
-
-static int
-connection_ready(struct ggd_connection *conn)
-{
-
- return (conn->c_sendfd != -1 && conn->c_recvfd != -1);
}
static void
@@ -742,7 +692,7 @@
conn = arg;
g_gate_log(LOG_NOTICE, "%s: started [%s]!", __func__, conn->c_path);
- fd = conn->c_recvfd;
+ fd = conn->c_netfd;
for (;;) {
/*
* Get header packet.
@@ -886,7 +836,7 @@
conn = arg;
g_gate_log(LOG_NOTICE, "%s: started [%s]!", __func__, conn->c_path);
- fd = conn->c_sendfd;
+ fd = conn->c_netfd;
for (;;) {
/*
* Get a request from the outgoing queue.
@@ -937,6 +887,9 @@
g_gate_log(LOG_INFO, "Connection from: %s.", ip2str(from));
}
+/*
+ * handshake return 1 if sfd is closed; otherwise 0 so that caller has to close.
+ */
static int
handshake(struct sockaddr *from, int sfd)
{
@@ -989,36 +942,23 @@
return (0);
}
g_gate_log(LOG_DEBUG, "Initial packet received.");
- conn = connection_find(&cinit);
- if (conn != NULL) {
- /*
- * Connection should already exists.
- */
- g_gate_log(LOG_DEBUG, "Found existing connection (token=%lu).",
- (unsigned long)conn->c_token);
- if (connection_add(conn, &cinit, from, sfd) == -1) {
- connection_remove(conn);
- return (0);
- }
- } else {
- /*
- * New connection, allocate space.
- */
- conn = connection_new(&cinit, from, sfd);
- if (conn == NULL) {
- sendfail(sfd, ENOMEM,
- "Cannot allocate new connection.");
- return (0);
- }
- g_gate_log(LOG_DEBUG, "New connection created (token=%lu).",
- (unsigned long)conn->c_token);
+
+ /*
+ * New connection, allocate space.
+ */
+ conn = connection_new(&cinit, from, sfd);
+ if (conn == NULL) {
+ sendfail(sfd, ENOMEM,
+ "Cannot allocate new connection.");
+ return (0);
}
+ g_gate_log(LOG_DEBUG, "New connection created (token=%lu).",
+ (unsigned long)conn->c_token);
ex = exports_find(from, &cinit, conn);
if (ex == NULL) {
- connection_remove(conn);
sendfail(sfd, errno, NULL);
- return (0);
+ goto out;
}
if (conn->c_mediasize == 0) {
conn->c_mediasize = g_gate_mediasize(conn->c_diskfd);
@@ -1033,16 +973,13 @@
g_gate_swap2n_sinit(&sinit);
data = g_gate_send(sfd, &sinit, sizeof(sinit), 0);
g_gate_swap2h_sinit(&sinit);
- if (data == -1) {
+ if (data == -1)
sendfail(sfd, errno, "Error while sending initial packet: %s.",
strerror(errno));
- return (0);
- }
-
- if (connection_ready(conn)) {
+ else
connection_launch(conn);
- connection_remove(conn);
- }
+out:
+ connection_remove(conn);
return (1);
}
diff -r 2fccee0290c5 -r bc62f8d4a1f0 shared/ggate.h
--- a/shared/ggate.h Tue Mar 17 14:17:39 2009 -0400
+++ b/shared/ggate.h Wed Mar 18 14:46:02 2009 -0400
@@ -41,18 +41,10 @@
#define G_GATE_TIMEOUT 0
#define GGATE_MAGIC "GEOM_GATE "
-#define GGATE_VERSION 0
+#define GGATE_VERSION 1
#define GGATE_FLAG_RDONLY 0x0001
#define GGATE_FLAG_WRONLY 0x0002
-/*
- * If GGATE_FLAG_SEND not GGATE_FLAG_RECV flag is set, this is initial
- * connection.
- * If GGATE_FLAG_SEND flag is set - this is socket to send data.
- * If GGATE_FLAG_RECV flag is set - this is socket to receive data.
- */
-#define GGATE_FLAG_SEND 0x0004
-#define GGATE_FLAG_RECV 0x0008
#define GGATE_CMD_READ 0
#define GGATE_CMD_WRITE 1
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200903190519.n2J5JDGM099384>