Date: Tue, 28 Jun 2011 19:27:34 +0000 (UTC) From: Mikolaj Golub <trociny@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-8@freebsd.org Subject: svn commit: r223651 - in stable/8/sbin: hastctl hastd Message-ID: <201106281927.p5SJRYK9062333@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: trociny Date: Tue Jun 28 19:27:34 2011 New Revision: 223651 URL: http://svn.freebsd.org/changeset/base/223651 Log: MFC r222164, r222228, r222467, r223181: r222164 (pjd): Recognize HIO_FLUSH requests. r222228 (pjd): Keep statistics on number of BIO_READ, BIO_WRITE, BIO_DELETE and BIO_FLUSH requests as well as number of activemap updates. Number of BIO_WRITEs and activemap updates are especially interesting, because if those two are too close to each other, it means that your workload needs bigger number of dirty extents. Activemap should be updated as rarely as possible. r222467: If READ from the local node failed we send the request to the remote node. There is no use in doing this for synchronization requests. r223181: In HAST we use two sockets - one for only sending the data and one for only receiving the data. In r220271 the unused directions were disabled using shutdown(2). Unfortunately, this broke automatic receive buffer sizing, which currently works only for connections in ETASBLISHED state. It was a root cause of the issue reported by users, when connection between primary and secondary could get stuck. Disable the code introduced in r220271 until the issue with automatic buffer sizing is not resolved. Reported by: Daniel Kalchev <daniel@digsys.bg>, danger, sobomax Tested by: Daniel Kalchev <daniel@digsys.bg>, danger Approved by: pjd (mentor) Modified: stable/8/sbin/hastctl/hastctl.c stable/8/sbin/hastd/control.c stable/8/sbin/hastd/hast.h stable/8/sbin/hastd/primary.c stable/8/sbin/hastd/secondary.c Directory Properties: stable/8/sbin/hastctl/ (props changed) stable/8/sbin/hastd/ (props changed) Modified: stable/8/sbin/hastctl/hastctl.c ============================================================================== --- stable/8/sbin/hastctl/hastctl.c Tue Jun 28 17:51:47 2011 (r223650) +++ stable/8/sbin/hastctl/hastctl.c Tue Jun 28 19:27:34 2011 (r223651) @@ -341,6 +341,17 @@ control_status(struct nv *nv) printf(" dirty: %ju (%NB)\n", (uintmax_t)nv_get_uint64(nv, "dirty%u", ii), (intmax_t)nv_get_uint64(nv, "dirty%u", ii)); + printf(" statistics:\n"); + printf(" reads: %ju\n", + (uint64_t)nv_get_uint64(nv, "stat_read%u", ii)); + printf(" writes: %ju\n", + (uint64_t)nv_get_uint64(nv, "stat_write%u", ii)); + printf(" deletes: %ju\n", + (uint64_t)nv_get_uint64(nv, "stat_delete%u", ii)); + printf(" flushes: %ju\n", + (uint64_t)nv_get_uint64(nv, "stat_flush%u", ii)); + printf(" activemap updates: %ju\n", + (uint64_t)nv_get_uint64(nv, "stat_activemap_update%u", ii)); } return (ret); } Modified: stable/8/sbin/hastd/control.c ============================================================================== --- stable/8/sbin/hastd/control.c Tue Jun 28 17:51:47 2011 (r223650) +++ stable/8/sbin/hastd/control.c Tue Jun 28 19:27:34 2011 (r223651) @@ -199,6 +199,16 @@ control_status_worker(struct hast_resour "extentsize%u", no); nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"), "keepdirty%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"), + "stat_read%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"), + "stat_write%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"), + "stat_delete%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"), + "stat_flush%u", no); + nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"), + "stat_activemap_update%u", no); end: if (cnvin != NULL) nv_free(cnvin); @@ -446,6 +456,13 @@ ctrl_thread(void *arg) nv_add_uint32(nvout, (uint32_t)0, "keepdirty"); nv_add_uint64(nvout, (uint64_t)0, "dirty"); } + nv_add_uint64(nvout, res->hr_stat_read, "stat_read"); + nv_add_uint64(nvout, res->hr_stat_write, "stat_write"); + nv_add_uint64(nvout, res->hr_stat_delete, + "stat_delete"); + nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush"); + nv_add_uint64(nvout, res->hr_stat_activemap_update, + "stat_activemap_update"); nv_add_int16(nvout, 0, "error"); break; case CONTROL_RELOAD: Modified: stable/8/sbin/hastd/hast.h ============================================================================== --- stable/8/sbin/hastd/hast.h Tue Jun 28 17:51:47 2011 (r223650) +++ stable/8/sbin/hastd/hast.h Tue Jun 28 19:27:34 2011 (r223651) @@ -218,6 +218,17 @@ struct hast_resource { /* Locked used to synchronize access to hr_amp. */ pthread_mutex_t hr_amp_lock; + /* Number of BIO_READ requests. */ + uint64_t hr_stat_read; + /* Number of BIO_WRITE requests. */ + uint64_t hr_stat_write; + /* Number of BIO_DELETE requests. */ + uint64_t hr_stat_delete; + /* Number of BIO_FLUSH requests. */ + uint64_t hr_stat_flush; + /* Number of activemap updates. */ + uint64_t hr_stat_activemap_update; + /* Next resource. */ TAILQ_ENTRY(hast_resource) hr_next; }; Modified: stable/8/sbin/hastd/primary.c ============================================================================== --- stable/8/sbin/hastd/primary.c Tue Jun 28 17:51:47 2011 (r223650) +++ stable/8/sbin/hastd/primary.c Tue Jun 28 19:27:34 2011 (r223651) @@ -726,11 +726,13 @@ init_remote(struct hast_resource *res, s (void)hast_activemap_flush(res); } nv_free(nvin); +#ifdef notyet /* Setup directions. */ if (proto_send(out, NULL, 0) == -1) pjdlog_errno(LOG_WARNING, "Unable to set connection direction"); if (proto_recv(in, NULL, 0) == -1) pjdlog_errno(LOG_WARNING, "Unable to set connection direction"); +#endif pjdlog_info("Connected to %s.", res->hr_remoteaddr); if (inp != NULL && outp != NULL) { *inp = in; @@ -1117,6 +1119,7 @@ ggate_recv_thread(void *arg) */ switch (ggio->gctl_cmd) { case BIO_READ: + res->hr_stat_read++; pjdlog_debug(2, "ggate_recv: (%p) Moving request to the send queue.", hio); @@ -1145,6 +1148,7 @@ ggate_recv_thread(void *arg) QUEUE_INSERT1(hio, send, ncomp); break; case BIO_WRITE: + res->hr_stat_write++; if (res->hr_resuid == 0) { /* * This is first write, initialize localcnt and @@ -1183,12 +1187,21 @@ ggate_recv_thread(void *arg) mtx_lock(&res->hr_amp_lock); if (activemap_write_start(res->hr_amp, ggio->gctl_offset, ggio->gctl_length)) { + res->hr_stat_activemap_update++; (void)hast_activemap_flush(res); } mtx_unlock(&res->hr_amp_lock); /* FALLTHROUGH */ case BIO_DELETE: case BIO_FLUSH: + switch (ggio->gctl_cmd) { + case BIO_DELETE: + res->hr_stat_delete++; + break; + case BIO_FLUSH: + res->hr_stat_flush++; + break; + } pjdlog_debug(2, "ggate_recv: (%p) Moving request to the send queues.", hio); @@ -1232,7 +1245,7 @@ local_send_thread(void *arg) ggio->gctl_offset + res->hr_localoff); if (ret == ggio->gctl_length) hio->hio_errors[ncomp] = 0; - else { + else if (!ISSYNCREQ(hio)) { /* * If READ failed, try to read from remote node. */ Modified: stable/8/sbin/hastd/secondary.c ============================================================================== --- stable/8/sbin/hastd/secondary.c Tue Jun 28 17:51:47 2011 (r223650) +++ stable/8/sbin/hastd/secondary.c Tue Jun 28 19:27:34 2011 (r223651) @@ -183,9 +183,11 @@ init_remote(struct hast_resource *res, s unsigned char *map; size_t mapsize; +#ifdef notyet /* Setup direction. */ if (proto_send(res->hr_remoteout, NULL, 0) == -1) pjdlog_errno(LOG_WARNING, "Unable to set connection direction"); +#endif map = NULL; mapsize = 0; @@ -351,9 +353,11 @@ init_remote(struct hast_resource *res, s if (map != NULL) free(map); nv_free(nvout); +#ifdef notyet /* Setup direction. */ if (proto_recv(res->hr_remotein, NULL, 0) == -1) pjdlog_errno(LOG_WARNING, "Unable to set connection direction"); +#endif if (res->hr_secondary_localcnt > res->hr_primary_remotecnt && res->hr_primary_localcnt > res->hr_secondary_remotecnt) { /* Exit on split-brain. */ @@ -514,6 +518,7 @@ requnpack(struct hast_resource *res, str goto end; } switch (hio->hio_cmd) { + case HIO_FLUSH: case HIO_KEEPALIVE: break; case HIO_READ: @@ -611,6 +616,20 @@ recv_thread(void *arg) QUEUE_INSERT(send, hio); continue; } + switch (hio->hio_cmd) { + case HIO_READ: + res->hr_stat_read++; + break; + case HIO_WRITE: + res->hr_stat_write++; + break; + case HIO_DELETE: + res->hr_stat_delete++; + break; + case HIO_FLUSH: + res->hr_stat_flush++; + break; + } reqlog(LOG_DEBUG, 2, -1, hio, "recv: (%p) Got request header: ", hio); if (hio->hio_cmd == HIO_KEEPALIVE) {
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201106281927.p5SJRYK9062333>