Date: Mon, 28 Feb 2011 09:11:46 +0000 (UTC) From: Poul-Henning Kamp <phk@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r219095 - in head/usr.sbin/fifolog: . fifolog_writer lib Message-ID: <201102280911.p1S9Bk76031590@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: phk Date: Mon Feb 28 09:11:46 2011 New Revision: 219095 URL: http://svn.freebsd.org/changeset/base/219095 Log: Update fifolog internals to match main version: Rename ...write_bytes... to ...write_records..., that's what they do. Move writer .h stuff into writer private .h file. Change logic in writer to support both fifolog usage in FreeBSD and Measured usage better, by always using an input buffer. Various cleanups. Modified: head/usr.sbin/fifolog/fifolog_writer/fifolog_writer.c head/usr.sbin/fifolog/flint.lnt head/usr.sbin/fifolog/lib/fifolog_create.c head/usr.sbin/fifolog/lib/fifolog_write.h head/usr.sbin/fifolog/lib/fifolog_write_poll.c head/usr.sbin/fifolog/lib/libfifolog.h head/usr.sbin/fifolog/lib/libfifolog_int.h Modified: head/usr.sbin/fifolog/fifolog_writer/fifolog_writer.c ============================================================================== --- head/usr.sbin/fifolog/fifolog_writer/fifolog_writer.c Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/fifolog_writer/fifolog_writer.c Mon Feb 28 09:11:46 2011 (r219095) @@ -42,7 +42,8 @@ static void usage(void) { - fprintf(stderr, "Usage: fifolog_writer [-w write-rate] [-s sync-rate] " + fprintf(stderr, + "Usage: fifolog_writer [-w write-rate] [-s sync-rate] " "[-z compression] file\n"); exit(EX_USAGE); } @@ -105,10 +106,10 @@ main(int argc, char * const *argv) p--; *p = '\0'; if (*buf != '\0') - fifolog_write_bytes_poll(f, 0, 0, buf, 0); + fifolog_write_record_poll(f, 0, 0, buf, 0); } else if (i == 0) - (void)fifolog_write_poll(f, 0); + fifolog_write_poll(f, 0); } - (void)fifolog_write_flush(f); + fifolog_write_close(f); return (0); } Modified: head/usr.sbin/fifolog/flint.lnt ============================================================================== --- head/usr.sbin/fifolog/flint.lnt Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/flint.lnt Mon Feb 28 09:11:46 2011 (r219095) @@ -1,49 +1,55 @@ // $FreeBSD$ // FlexeLint file for fifolog tools // - --passes=3 --ffc - -// GCC --cgnu -+d__FreeBSD__=7 -+d__GNUC__=4 -+d__GNUC_MINOR__=2 -+d__FreeBSD_cc_version=700003 -+d__attribute__()= --d__builtin_va_list=void* // used by stdarg.h -// -d__builtin_stdarg_start()=_to_semi // ditto -// -d__builtin_va_start(a,b)=((void)(b),(a)=0) // ditto -// -d__builtin_va_end()=_to_semi // ditto -+rw(__inline) // enable the (non-standard) __inline keyword -+rw(__inline__) // enable the (non-standard) __inline__ keyword - -+d"__unused=/*lint -e{715} -e{818} */" - --e537 // Repeated include file --elib(652) // #define of symbol '...' declared previously --function(exit,__assert) --function(exit,err) --function(exit,errx) --e716 // while(1) ... --e717 // do ... while(0) - -// Ignore return values +// +//-passes=3 +//-ffc +// +//// GCC +//-cgnu +//+d__FreeBSD__=7 +//+d__GNUC__=4 +//+d__GNUC_MINOR__=2 +//+d__FreeBSD_cc_version=700003 +//+d__attribute__()= +//-d__builtin_va_list=void* // used by stdarg.h +//// -d__builtin_stdarg_start()=_to_semi // ditto +//// -d__builtin_va_start(a,b)=((void)(b),(a)=0) // ditto +//// -d__builtin_va_end()=_to_semi // ditto +//+rw(__inline) // enable the (non-standard) __inline keyword +//+rw(__inline__) // enable the (non-standard) __inline__ keyword +// +//+d"__unused=/*lint -e{715} -e{818} */" +// +//-e537 // Repeated include file +//-elib(652) // #define of symbol '...' declared previously +//-function(exit,__assert) +//-function(exit,err) +//-function(exit,errx) +//-e716 // while(1) ... +//-e717 // do ... while(0) +// +//// Ignore return values -esym(534, memset) -esym(534, memcpy) -esym(534, strcpy) --esym(534, printf) +//-esym(534, printf) -esym(534, time) -esym(534, fprintf) --esym(534, vfprintf) - -+libh(fifolog.h) -+libh(miniobj.h) -+libh(libfifolog.h) +//-esym(534, vfprintf) +// +//+libh(fifolog.h) +//+libh(miniobj.h) +//+libh(libfifolog.h) +// +//-e713 // loss of precision sign/unsigned +//-e732 // loss of sign +//-e734 // loss of precision assignment +//-e737 // loss of sign in promotion int->unsigned +//-e573 // sign/unsign mix in divide --e713 // loss of precision sign/unsigned --e732 // loss of sign --e734 // loss of precision assignment --e737 // loss of sign in promotion int->unsigned --e573 // sign/unsign mix in divide +-e712 // Loss of precision (___) (___ to ___) +-e713 // Loss of precision (___) (___ to ___) +-e716 // while(1) ... +-e732 // Loss of sign (___) (___ to ___) +-e747 // Significant prototype coercion (___) ___ to ___ Modified: head/usr.sbin/fifolog/lib/fifolog_create.c ============================================================================== --- head/usr.sbin/fifolog/lib/fifolog_create.c Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/lib/fifolog_create.c Mon Feb 28 09:11:46 2011 (r219095) @@ -41,7 +41,7 @@ #include "libfifolog.h" const char * -fifolog_create(const char *fn, off_t size, unsigned recsize) +fifolog_create(const char *fn, off_t size, ssize_t recsize) { int i, fd; unsigned u; @@ -103,7 +103,7 @@ fifolog_create(const char *fn, off_t siz strcpy(buf, FIFOLOG_FMT_MAGIC); /*lint !e64 */ be32enc(buf + FIFOLOG_OFF_BS, recsize); - if ((int)recsize != pwrite(fd, buf, recsize, 0)) { + if (recsize != pwrite(fd, buf, recsize, 0)) { i = errno; free(buf); errno = i; Modified: head/usr.sbin/fifolog/lib/fifolog_write.h ============================================================================== --- head/usr.sbin/fifolog/lib/fifolog_write.h Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/lib/fifolog_write.h Mon Feb 28 09:11:46 2011 (r219095) @@ -50,16 +50,26 @@ struct fifolog_writer { uint32_t seq; off_t recno; - int flag; + uint8_t flag; time_t last; - u_int obufsize; + ssize_t obufsize; u_char *obuf; - u_int ibufsize; + ssize_t ibufsize; + ssize_t ibufptr; u_char *ibuf; time_t starttime; time_t lastwrite; time_t lastsync; }; + +struct fifolog_writer *fifolog_write_new(void); +const char *fifolog_write_open(struct fifolog_writer *f, const char *fn, unsigned writerate, unsigned syncrate, unsigned compression); +int fifolog_write_record(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, ssize_t len); +int fifolog_write_poll(struct fifolog_writer *f, time_t now); +int fifolog_write_record_poll(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, ssize_t len); +void fifolog_write_close(struct fifolog_writer *f); +void fifolog_write_destroy(struct fifolog_writer *f); +extern const char *fifolog_write_statnames[]; Modified: head/usr.sbin/fifolog/lib/fifolog_write_poll.c ============================================================================== --- head/usr.sbin/fifolog/lib/fifolog_write_poll.c Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/lib/fifolog_write_poll.c Mon Feb 28 09:11:46 2011 (r219095) @@ -31,20 +31,19 @@ #include <string.h> #include <stdlib.h> #include <unistd.h> +#include <stdint.h> #include <time.h> #include <sys/endian.h> -#if 0 -#include <sys/uio.h> -#endif #include <zlib.h> #include "fifolog.h" -#include "libfifolog.h" #include "libfifolog_int.h" #include "fifolog_write.h" #include "miniobj.h" +static int fifolog_write_gzip(struct fifolog_writer *f, time_t now); + #define ALLOC(ptr, size) do { \ (*(ptr)) = calloc(size, 1); \ assert(*(ptr) != NULL); \ @@ -60,7 +59,7 @@ const char *fifolog_write_statnames[] = [FIFOLOG_PT_RUNTIME] = "Runtime" }; -/* +/********************************************************************** * Check that everything is all right */ static void @@ -72,6 +71,10 @@ fifolog_write_assert(const struct fifolo f->obuf + f->obufsize); } +/********************************************************************** + * Allocate/Destroy a new fifolog writer instance + */ + struct fifolog_writer * fifolog_write_new(void) { @@ -85,24 +88,35 @@ fifolog_write_new(void) void fifolog_write_destroy(struct fifolog_writer *f) { - CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); - free(f); + + free(f->obuf); + free(f->ibuf); + FREE_OBJ(f); } +/********************************************************************** + * Open/Close the fifolog + */ + void fifolog_write_close(struct fifolog_writer *f) { + time_t now; CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); + fifolog_write_assert(f); + + f->cleanup = 1; + time(&now); + fifolog_write_gzip(f, now); + fifolog_write_assert(f); fifolog_int_close(&f->ff); free(f->ff); - if (f->obuf != NULL) - free(f->obuf); - free(f); } const char * -fifolog_write_open(struct fifolog_writer *f, const char *fn, unsigned writerate, unsigned syncrate, int compression) +fifolog_write_open(struct fifolog_writer *f, const char *fn, + unsigned writerate, unsigned syncrate, unsigned compression) { const char *es; int i; @@ -112,8 +126,7 @@ fifolog_write_open(struct fifolog_writer CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); /* Check for legal compression value */ - if (compression < Z_DEFAULT_COMPRESSION || - compression > Z_BEST_COMPRESSION) + if (compression > Z_BEST_COMPRESSION) return ("Illegal compression value"); f->writerate = writerate; @@ -144,6 +157,10 @@ fifolog_write_open(struct fifolog_writer f->obufsize = f->ff->recsize; ALLOC(&f->obuf, f->obufsize); + f->ibufsize = f->obufsize * 10; + ALLOC(&f->ibuf, f->ibufsize); + f->ibufptr = 0; + i = deflateInit(f->ff->zs, (int)f->compression); assert(i == Z_OK); @@ -161,11 +178,17 @@ fifolog_write_open(struct fifolog_writer return (NULL); } +/********************************************************************** + * Write an output record + * Returns -1 if there are trouble writing data + */ + static int fifolog_write_output(struct fifolog_writer *f, int fl, time_t now) { long h, l = f->ff->zs->next_out - f->obuf; - int i, w; + ssize_t i, w; + int retval = 0; h = 4; /* seq */ be32enc(f->obuf, f->seq); @@ -176,13 +199,14 @@ fifolog_write_output(struct fifolog_writ h += 4; /* timestamp */ } - assert(l <= (long)f->ff->recsize); + assert(l <= (long)f->ff->recsize); /* NB: l includes h */ assert(l >= h); + + /* We will never write an entirely empty buffer */ if (l == h) return (0); - - if (h + l < (long)f->ff->recsize && fl == Z_NO_FLUSH) + if (l < (long)f->ff->recsize && fl == Z_NO_FLUSH) return (0); w = f->ff->recsize - l; @@ -190,117 +214,111 @@ fifolog_write_output(struct fifolog_writ be32enc(f->obuf + f->ff->recsize - 4, w); f->obuf[4] |= FIFOLOG_FLG_4BYTE; } else if (w > 0) { - f->obuf[f->ff->recsize - 1] = w; + f->obuf[f->ff->recsize - 1] = (uint8_t)w; f->obuf[4] |= FIFOLOG_FLG_1BYTE; } - f->cnt[FIFOLOG_PT_BYTES_POST] += w; - -#ifdef DBG -fprintf(stderr, "W: fl=%d h=%ld l=%ld w=%d recno=%jd fx %02x\n", - fl, h, l, w, f->recno, f->obuf[4]); -#endif + f->cnt[FIFOLOG_PT_BYTES_POST] += l - h; i = pwrite(f->ff->fd, f->obuf, f->ff->recsize, (f->recno + 1) * f->ff->recsize); - assert(i == (int)f->ff->recsize); + if (i != f->ff->recsize) + retval = -1; + else + retval = 1; f->cnt[FIFOLOG_PT_WRITES]++; + f->cnt[FIFOLOG_PT_RUNTIME] = now - f->starttime; f->lastwrite = now; + /* + * We increment these even on error, so as to properly skip bad, + * sectors or other light trouble. + */ f->seq++; f->recno++; -#ifdef DBG -if (f->flag) -fprintf(stderr, "SYNC- %d\n", __LINE__); -#endif f->flag = 0; memset(f->obuf, 0, f->obufsize); f->ff->zs->next_out = f->obuf + 5; f->ff->zs->avail_out = f->obufsize - 5; - return (1); + return (retval); } -static void -fifolog_write_gzip(struct fifolog_writer *f, const void *p, int len, time_t now, int fin) -{ - int i, fl; +/********************************************************************** + * Run the compression engine + * Returns -1 if there are trouble writing data + */ - f->cnt[FIFOLOG_PT_BYTES_PRE] += len; +static int +fifolog_write_gzip(struct fifolog_writer *f, time_t now) +{ + int i, fl, retval = 0; - if (fin == 0) - fl = Z_NO_FLUSH; - else if (f->cleanup || now >= (int)(f->lastsync + f->syncrate)) { + assert(now != 0); + if (f->cleanup || now >= (int)(f->lastsync + f->syncrate)) { f->cleanup = 0; fl = Z_FINISH; f->cnt[FIFOLOG_PT_SYNC]++; } else if (now >= (int)(f->lastwrite + f->writerate)) { fl = Z_SYNC_FLUSH; f->cnt[FIFOLOG_PT_FLUSH]++; - } else if (p == NULL) - return; + } else if (f->ibufptr == 0) + return (0); else fl = Z_NO_FLUSH; - f->ff->zs->avail_in = len; - f->ff->zs->next_in = (void*)(uintptr_t)p; -#ifdef DBG -if (fl != Z_NO_FLUSH) -fprintf(stderr, "Z len %3d fin %d now %ld fl %d ai %u ao %u\n", - len, fin, now, fl, - f->ff->zs->avail_in, - f->ff->zs->avail_out); -#endif + f->ff->zs->avail_in = f->ibufptr; + f->ff->zs->next_in = f->ibuf; while (1) { i = deflate(f->ff->zs, fl); - -#ifdef DBG -if (i || f->ff->zs->avail_in) -fprintf(stderr, "fl = %d, i = %d ai = %u ao = %u fx=%02x\n", fl, i, - f->ff->zs->avail_in, - f->ff->zs->avail_out, f->flag); -#endif - assert(i == Z_OK || i == Z_BUF_ERROR || i == Z_STREAM_END); - assert(f->ff->zs->avail_in == 0); - if (!fifolog_write_output(f, fl, now)) + i = fifolog_write_output(f, fl, now); + if (i == 0) break; + if (i < 0) + retval = -1; } assert(f->ff->zs->avail_in == 0); + f->ibufptr = 0; if (fl == Z_FINISH) { f->flag |= FIFOLOG_FLG_SYNC; f->ff->zs->next_out = f->obuf + 9; f->ff->zs->avail_out = f->obufsize - 9; f->lastsync = now; -#ifdef DBG -fprintf(stderr, "SYNC %d\n", __LINE__); -#endif assert(Z_OK == deflateReset(f->ff->zs)); } + return (retval); } +/********************************************************************** + * Poll to see if we need to flush out a record + * Returns -1 if there are trouble writing data + */ + int fifolog_write_poll(struct fifolog_writer *f, time_t now) { + if (now == 0) time(&now); - fifolog_write_gzip(f, NULL, 0, now, 1); - return (0); + return (fifolog_write_gzip(f, now)); } -/* - * Attempt to write an entry. +/********************************************************************** + * Attempt to write an entry into the ibuf. * Return zero if there is no space, one otherwise */ int -fifolog_write_bytes(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, unsigned len) +fifolog_write_record(struct fifolog_writer *f, uint32_t id, time_t now, + const void *ptr, ssize_t len) { const unsigned char *p; - uint8_t buf[4]; + uint8_t buf[9]; + unsigned bufl; fifolog_write_assert(f); assert(!(id & (FIFOLOG_TIMESTAMP|FIFOLOG_LENGTH))); @@ -308,94 +326,87 @@ fifolog_write_bytes(struct fifolog_write p = ptr; if (len == 0) { - len = strlen(ptr) + 1; + len = strlen(ptr); + len++; } else { assert(len <= 255); id |= FIFOLOG_LENGTH; } + assert (len > 0); - /* Now do timestamp, if needed */ + /* Do a timestamp, if needed */ if (now == 0) time(&now); - if (now != f->last) { + if (now != f->last) id |= FIFOLOG_TIMESTAMP; - f->last = now; - } /* Emit instance+flag */ be32enc(buf, id); - fifolog_write_gzip(f, buf, 4, now, 0); + bufl = 4; if (id & FIFOLOG_TIMESTAMP) { - be32enc(buf, (uint32_t)f->last); - fifolog_write_gzip(f, buf, 4, now, 0); - } - if (id & FIFOLOG_LENGTH) { - buf[0] = (u_char)len; - fifolog_write_gzip(f, buf, 1, now, 0); + be32enc(buf + bufl, (uint32_t)now); + bufl += 4; } + if (id & FIFOLOG_LENGTH) + buf[bufl++] = (u_char)len; - assert (len > 0); -#if 1 - if (len > f->ibufsize) { - free(f->ibuf); - f->ibufsize = len; - ALLOC(&f->ibuf, f->ibufsize); - } - memcpy(f->ibuf, p, len); - fifolog_write_gzip(f, f->ibuf, len, now, 1); -#else - fifolog_write_gzip(f, p, len, now, 1); -#endif - fifolog_write_assert(f); + if (bufl + len + f->ibufptr > f->ibufsize) + return (0); + + memcpy(f->ibuf + f->ibufptr, buf, bufl); + f->ibufptr += bufl; + memcpy(f->ibuf + f->ibufptr, p, len); + f->ibufptr += len; + f->cnt[FIFOLOG_PT_BYTES_PRE] += bufl + len; + + if (id & FIFOLOG_TIMESTAMP) + f->last = now; return (1); } -/* - * Write an entry, polling until success. +/********************************************************************** + * Write an entry, polling the gzip/writer until success. * Long binary entries are broken into 255 byte chunks. + * Returns -1 if there are problems writing data */ -void -fifolog_write_bytes_poll(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, unsigned len) +int +fifolog_write_record_poll(struct fifolog_writer *f, uint32_t id, time_t now, + const void *ptr, ssize_t len) { u_int l; const unsigned char *p; + int retval = 0; + if (now == 0) + time(&now); fifolog_write_assert(f); assert(!(id & (FIFOLOG_TIMESTAMP|FIFOLOG_LENGTH))); assert(ptr != NULL); if (len == 0) { - while (!fifolog_write_bytes(f, id, now, ptr, len)) { - (void)usleep(10000); + if (!fifolog_write_record(f, id, now, ptr, len)) { + if (fifolog_write_gzip(f, now) < 0) + retval = -1; + /* The string could be too long for the ibuf, so... */ + if (!fifolog_write_record(f, id, now, ptr, len)) + retval = -1; } } else { - p = ptr; for (p = ptr; len > 0; len -= l, p += l) { l = len; if (l > 255) l = 255; - while (!fifolog_write_bytes(f, id, now, p, l)) { - (void)usleep(10000); - } + while (!fifolog_write_record(f, id, now, p, l)) + if (fifolog_write_gzip(f, now) < 0) + retval = -1; } } + if (fifolog_write_gzip(f, now) < 0) + retval = -1; fifolog_write_assert(f); -} - -int -fifolog_write_flush(struct fifolog_writer *f) -{ - int i; - - fifolog_write_assert(f); - - f->cleanup = 1; - for (i = 0; fifolog_write_poll(f, 0); i = 1) - continue; - fifolog_write_assert(f); - return (i); + return (retval); } Modified: head/usr.sbin/fifolog/lib/libfifolog.h ============================================================================== --- head/usr.sbin/fifolog/lib/libfifolog.h Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/lib/libfifolog.h Mon Feb 28 09:11:46 2011 (r219095) @@ -29,21 +29,11 @@ #include <sys/stdint.h> /* CREATORS */ -const char *fifolog_create(const char *fn, off_t size, unsigned recsize); - +const char *fifolog_create(const char *fn, off_t size, ssize_t recsize); /* WRITERS */ -struct fifolog_writer; -struct fifolog_writer *fifolog_write_new(void); -const char *fifolog_write_open(struct fifolog_writer *f, const char *fn, unsigned writerate, unsigned syncrate, int compression); -int fifolog_write_bytes(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, unsigned len); -void fifolog_write_bytes_poll(struct fifolog_writer *f, uint32_t id, time_t now, const void *ptr, unsigned len); -int fifolog_write_poll(struct fifolog_writer *f, time_t now); -void fifolog_write_close(struct fifolog_writer *f); -void fifolog_write_destroy(struct fifolog_writer *f); -int fifolog_write_flush(struct fifolog_writer *f); -extern const char *fifolog_write_statnames[]; +#include "fifolog_write.h" /* READERS */ @@ -55,8 +45,3 @@ void fifolog_reader_process(struct fifol /* UTILS */ time_t get_date(char *p); - -#if (__FreeBSD__ < 7) -int expand_number(char *_buf, int64_t *_num); -#endif - Modified: head/usr.sbin/fifolog/lib/libfifolog_int.h ============================================================================== --- head/usr.sbin/fifolog/lib/libfifolog_int.h Mon Feb 28 08:59:35 2011 (r219094) +++ head/usr.sbin/fifolog/lib/libfifolog_int.h Mon Feb 28 09:11:46 2011 (r219095) @@ -30,7 +30,7 @@ struct fifolog_file { unsigned magic; #define FIFOLOG_FILE_MAGIC 0x307ea50d - unsigned recsize; + ssize_t recsize; off_t logsize; int fd;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201102280911.p1S9Bk76031590>