Date: Tue, 16 Nov 2010 23:39:26 +0200 From: Mikolaj Golub <to.my.trociny@gmail.com> To: freebsd-fs@freebsd.org Subject: HAST: more than one remote nodes, is it possible? Message-ID: <86eial2bjl.fsf@kopusha.home.net>
next in thread | raw e-mail | index | archive | help
--=-=-=
Hi,
So is it possible to have several remote (secondary) nodes with HAST? Although
current version of hastd supports only one remote node, it looks like it is
possible building HAST devices in stack :-)
I tried this under VirtulBox and it worked for me.
Having 2 pairs of hosts I built lower HAST devise on both pairs and then upper
resource using lower as a provider for upper. Any host can be used as primary
for upper in this configuration. To make things more interesting I installed
jail (VIMAGE) on upper device so vm can be moved between 4 hosts.
Below is hast.conf:
----------------------------------------------------------------------
# Stacked HAST -- data are written to 4 nodes (lolek, bolek, chuk and
# gek) and are accessible on one of the nodes as /dev/hast/upper
#
# /dev/hast/upper
# |
# P------------upper-------------S
# | 172.20.68.30 |
# /dev/hast/lower /dev/hast/lower
# | |
# P-----lower----S P----lower-----S
# | 172.20.68.31 | | 172.20.68.32 |
# /dev/ad4 /dev/ad4 /dev/ad4 /dev/ad4
#
# lolek bolek chuk gek
exec /etc/vip.sh
resource lower {
local /dev/ad4
on lolek {
remote bolek
}
on bolek {
remote lolek
}
on chuk {
remote gek
}
on gek {
remote chuk
}
}
resource upper {
local /dev/hast/lower
# XXX: HAST should be patched to know friends
friends lolek bolek chuk gek
on lolek {
remote 172.20.68.32
}
on bolek {
remote 172.20.68.32
}
on chuk {
remote 172.20.68.31
}
on gek {
remote 172.20.68.31
}
}
----------------------------------------------------------------------
As it is noted in the comments above hastd should be patched: current version
of hastd allows only connections from addresses specified as remote. The patch
is attached.
/etc/vip.sh is used to switch IP addresses between hosts.
Details can be found here:
http://code.google.com/p/hastmon/wiki/StackedHAST
--
Mikolaj Golub
--=-=-=
Content-Type: text/x-patch
Content-Disposition: attachment; filename=hastd.friends.patch
Index: sbin/hastd/parse.y
===================================================================
--- sbin/hastd/parse.y (revision 214604)
+++ sbin/hastd/parse.y (working copy)
@@ -209,8 +209,13 @@ void
yy_config_free(struct hastd_config *config)
{
struct hast_resource *res;
+ struct hast_address *addr;
while ((res = TAILQ_FIRST(&config->hc_resources)) != NULL) {
+ while ((addr = TAILQ_FIRST(&res->hr_friends)) != NULL) {
+ TAILQ_REMOVE(&res->hr_friends, addr, ha_next);
+ free(addr);
+ }
TAILQ_REMOVE(&config->hc_resources, res, hr_next);
free(res);
}
@@ -218,7 +223,8 @@ yy_config_free(struct hastd_config *config)
}
%}
-%token CONTROL LISTEN PORT REPLICATION TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE ON
+%token CONTROL LISTEN PORT REPLICATION TIMEOUT EXEC EXTENTSIZE
+%token RESOURCE NAME LOCAL REMOTE FRIENDS ON
%token FULLSYNC MEMSYNC ASYNC
%token NUM STR OB CB
@@ -418,6 +424,8 @@ node_entry:
control_statement
|
listen_statement
+ |
+ friends_statement
;
resource_statement: RESOURCE resource_start OB resource_entries CB
@@ -513,6 +521,7 @@ resource_start: STR
curres->hr_localpath[0] = '\0';
curres->hr_localfd = -1;
curres->hr_remoteaddr[0] = '\0';
+ TAILQ_INIT(&curres->hr_friends);
curres->hr_ggateunit = -1;
}
;
@@ -533,6 +542,8 @@ resource_entry:
|
local_statement
|
+ friends_statement
+ |
resource_node_statement
;
@@ -598,6 +609,40 @@ local_statement: LOCAL STR
}
;
+friends_statement: FRIENDS friend_addresses
+ ;
+
+friend_addresses:
+ |
+ friend_addresses friend_address
+ ;
+
+friend_address: STR
+ {
+ struct hast_address *addr;
+
+ assert(depth == 1 || depth == 2);
+
+ if (depth == 1 || mynode) {
+ assert(curres != NULL);
+ addr = calloc(1, sizeof(*addr));
+ if (addr == NULL) {
+ errx(EX_TEMPFAIL,
+ "cannot allocate memory for resource");
+ }
+ if (strlcpy(addr->ha_addr, $1,
+ sizeof(addr->ha_addr)) >=
+ sizeof(addr->ha_addr)) {
+ pjdlog_error("address argument too long");
+ free($1);
+ return (1);
+ }
+ free($1);
+ TAILQ_INSERT_TAIL(&curres->hr_friends, addr, ha_next);
+ }
+ }
+ ;
+
resource_node_statement:ON resource_node_start OB resource_node_entries CB
{
mynode = false;
Index: sbin/hastd/hastd.c
===================================================================
--- sbin/hastd/hastd.c (revision 214604)
+++ sbin/hastd/hastd.c (working copy)
@@ -156,6 +156,29 @@ child_exit(void)
}
}
+static int
+friendscmp(const struct hast_resource *res0, const struct hast_resource *res1)
+{
+ struct hast_address *friend0, *friend1;
+ int nfriends0, nfriends1;
+
+ nfriends0 = nfriends1 = 0;
+
+ TAILQ_FOREACH(friend0, &res0->hr_friends, ha_next) {
+ TAILQ_FOREACH(friend1, &res1->hr_friends, ha_next) {
+ if (strcmp(friend0->ha_addr, friend1->ha_addr) == 0)
+ break;
+ }
+ if (friend1 == NULL)
+ return (1);
+ nfriends0++;
+ }
+ TAILQ_FOREACH(friend1, &res1->hr_friends, ha_next) {
+ nfriends1++;
+ }
+ return (nfriends0 - nfriends1);
+}
+
static bool
resource_needs_restart(const struct hast_resource *res0,
const struct hast_resource *res1)
@@ -177,6 +200,8 @@ resource_needs_restart(const struct hast_resource
return (true);
if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
return (true);
+ if (friendscmp(res0, res1) != 0)
+ return (true);
}
return (false);
}
@@ -201,6 +226,7 @@ resource_needs_reload(const struct hast_resource *
return (true);
if (strcmp(res0->hr_exec, res1->hr_exec) != 0)
return (true);
+
return (false);
}
@@ -384,6 +410,7 @@ listen_accept(void)
struct hast_resource *res;
struct proto_conn *conn;
struct nv *nvin, *nvout, *nverr;
+ struct hast_address *friend;
const char *resname;
const unsigned char *token;
char laddr[256], raddr[256];
@@ -416,6 +443,12 @@ listen_accept(void)
TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
if (proto_address_match(conn, res->hr_remoteaddr))
break;
+ TAILQ_FOREACH(friend, &res->hr_friends, ha_next) {
+ if (proto_address_match(conn, friend->ha_addr))
+ break;
+ }
+ if (friend != NULL)
+ break;
}
if (res == NULL) {
pjdlog_error("Client %s isn't known.", raddr);
@@ -469,9 +502,15 @@ listen_accept(void)
/* Does the remote host have access to this resource? */
if (!proto_address_match(conn, res->hr_remoteaddr)) {
- pjdlog_error("Client %s has no access to the resource.", raddr);
- nv_add_stringf(nverr, "errmsg", "No access to the resource.");
- goto fail;
+ TAILQ_FOREACH(friend, &res->hr_friends, ha_next) {
+ if (proto_address_match(conn, friend->ha_addr))
+ break;
+ }
+ if (friend == NULL) {
+ pjdlog_error("Client %s has no access to the resource.", raddr);
+ nv_add_stringf(nverr, "errmsg", "No access to the resource.");
+ goto fail;
+ }
}
/* Is the resource marked as secondary? */
if (res->hr_role != HAST_ROLE_SECONDARY) {
Index: sbin/hastd/hast.h
===================================================================
--- sbin/hastd/hast.h (revision 214604)
+++ sbin/hastd/hast.h (working copy)
@@ -150,6 +150,8 @@ struct hast_resource {
/* Address of the remote component. */
char hr_remoteaddr[HAST_ADDRSIZE];
+ /* Hosts that can connect to us. */
+ TAILQ_HEAD(, hast_address) hr_friends;
/* Connection for incoming data. */
struct proto_conn *hr_remotein;
/* Connection for outgoing data. */
@@ -193,6 +195,11 @@ struct hast_resource {
TAILQ_ENTRY(hast_resource) hr_next;
};
+struct hast_address {
+ char ha_addr[HAST_ADDRSIZE];
+ TAILQ_ENTRY(hast_address) ha_next;
+};
+
struct hastd_config *yy_config_parse(const char *config, bool exitonerror);
void yy_config_free(struct hastd_config *config);
Index: sbin/hastd/hast.conf.5
===================================================================
--- sbin/hastd/hast.conf.5 (revision 214604)
+++ sbin/hastd/hast.conf.5 (working copy)
@@ -81,6 +81,7 @@ resource <name> {
local <path>
timeout <seconds>
exec <path>
+ friends <addr ...>
on <node> {
# Resource-node section
@@ -89,6 +90,7 @@ resource <name> {
local <path>
# Required
remote <addr>
+ friends <addr ...>
}
on <node> {
# Resource-node section
@@ -97,6 +99,7 @@ resource <name> {
local <path>
# Required
remote <addr>
+ friends <addr ...>
}
}
.Ed
@@ -155,6 +158,13 @@ tcp4://0.0.0.0:8457
.Pp
The default value is
.Pa tcp4://0.0.0.0:8457 .
+.It Ic friends Aq addr ...
+.Pp
+List of addresses (separated by space) of hosts that can connect to
+the node.
+Format is the same as for the
+.Ic listen
+statement.
.It Ic replication Aq mode
.Pp
Replication mode should be one of the following:
Index: sbin/hastd/token.l
===================================================================
--- sbin/hastd/token.l (revision 214604)
+++ sbin/hastd/token.l (working copy)
@@ -53,6 +53,7 @@ exec { DP; return EXEC; }
resource { DP; return RESOURCE; }
name { DP; return NAME; }
local { DP; return LOCAL; }
+friends { DP; return FRIENDS; }
remote { DP; return REMOTE; }
on { DP; return ON; }
fullsync { DP; return FULLSYNC; }
--=-=-=--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?86eial2bjl.fsf>