Date: Thu, 5 Apr 2012 07:24:58 +0000 (UTC) From: Jeremie Le Hen <jeremie@le-hen.org> To: FreeBSD-gnats-submit@FreeBSD.org Subject: bin/166660: [patch] New util/shlib to change per-fd default stdio buffering mode Message-ID: <20120405072458.6E55DD95F@felucia.tataz.chchile.org> Resent-Message-ID: <201204050730.q357U29P038825@freefall.freebsd.org>
next in thread | raw e-mail | index | archive | help
>Number: 166660
>Category: bin
>Synopsis: [patch] New util/shlib to change per-fd default stdio buffering mode
>Confidential: no
>Severity: non-critical
>Priority: low
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: change-request
>Submitter-Id: current-users
>Arrival-Date: Thu Apr 05 07:30:02 UTC 2012
>Closed-Date:
>Last-Modified:
>Originator: Jeremie Le Hen
>Release: FreeBSD 9.0-STABLE
>Organization:
>Environment:
System: FreeBSD 9.0-STABLE
>Description:
When stdin and stdout are not connected to a terminal, the stdio
subsystem switches automatically stdout buffering mode from
line-buffered to fully-buffered.
This is a problem when trying to watch the real-time output of a
command such as vmstat(8) or iostat(8) through multiple filters.
For instance: iostat -x 1 | cat -n | grep -v "extended"
In this case, cat(1)'s stdout is fully buffered and iostat(8)
produces output too slowly, so you have to wait multiple seconds
before the buffer is eventually flushed as a block to the last
command.
>How-To-Repeat:
Just run the command above. On my system with 4 drives, I have
to wait more than 20 seconds to see something.
>Fix:
The patch below adds a new shared library "libstdbuf.so" which,
when LD_PRELOAD'ed, can be configured through environment
variables to override the default buffering mode of either
stdin, stdout or stderr.
In order to avoid the manual setting of LD_PRELOAD and configuration
variables, an utility is provided: stdbuf(1). It is named after a
similar functionality found on Linux. Of course, given the
command-line interface is sane enough to comply with BSD standards,
I kept it fully compatible.
Here is an example of how the above problem can be solved with this
new utility:
iostat -x 1 | stdbuf -o L cat -n | grep -v "extended"
--- stdbuf.diff begins here ---
diff -urNp src.HEAD_20111506/lib/libc/stdio/setbuf.3 src/lib/libc/stdio/setbuf.3
--- src.HEAD_20111506/lib/libc/stdio/setbuf.3 2007-01-09 01:28:07.000000000 +0100
+++ src/lib/libc/stdio/setbuf.3 2011-08-04 19:00:49.000000000 +0200
@@ -83,6 +83,9 @@ normally does) it is line buffered.
The standard error stream
.Dv stderr
is always unbuffered.
+Note that these defaults maybe be altered using the
+.Xr stdbuf 1
+utility.
.Pp
The
.Fn setvbuf
@@ -177,6 +180,7 @@ function returns what the equivalent
.Fn setvbuf
would have returned.
.Sh SEE ALSO
+.Xr stdbuf 1 ,
.Xr fclose 3 ,
.Xr fopen 3 ,
.Xr fread 3 ,
diff -urNp src.HEAD_20111506/lib/libstdbuf/Makefile src/lib/libstdbuf/Makefile
--- src.HEAD_20111506/lib/libstdbuf/Makefile 1970-01-01 01:00:00.000000000 +0100
+++ src/lib/libstdbuf/Makefile 2011-08-04 18:49:59.000000000 +0200
@@ -0,0 +1,15 @@
+# $FreeBSD: src/lib/libftpio/Makefile,v 1.17.2.1 2009/08/03 08:13:06 kensmith Exp $
+
+.include <bsd.own.mk>
+
+LIB= stdbuf
+SRCS= stdbuf.c
+SHLIB_MAJOR= 1
+MAN= libstdbuf.3
+
+CFLAGS+= -I${.CURDIR}
+LDADD= -lutil
+
+WARNS?= 6
+
+.include <bsd.lib.mk>
diff -urNp src.HEAD_20111506/lib/libstdbuf/libstdbuf.3 src/lib/libstdbuf/libstdbuf.3
--- src.HEAD_20111506/lib/libstdbuf/libstdbuf.3 1970-01-01 01:00:00.000000000 +0100
+++ src/lib/libstdbuf/libstdbuf.3 2011-08-04 18:47:04.000000000 +0200
@@ -0,0 +1,111 @@
+.\" Copyright (c) 2011 Jeremie Le Hen
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\" notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd August 4, 2011
+.Dt LIBSTDBUF 3
+.Os
+.Sh NAME
+.Nm libstdbuf
+.Nd preloaded library to change standard streams initial buffering
+.Sh DESCRIPTION
+The
+.Nm
+library is meant to be preloaded with the
+.Ev LD_PRELOAD
+environment variable to as to change the initial buffering
+of standard input, standard output and standard error streams.
+.Pp
+Although you may load and configure this library manually,
+an utility,
+.Xr stdbuf 1 ,
+can be used to run a command with the appropriate environment variables.
+.Sh ENVIRONMENT
+Each stream can be configured indepentently through the following
+environment variables (values are defined below):
+.Bl -tag -width size -offset indent
+.It Ev STDBUF_0
+Initial buffering definition for the standard input stream
+.It Ev STDBUF_1
+Initial buffering definition for the standard output stream
+.It Ev STDBUF_2
+Initial buffering definition for the standard error stream
+.It Ev _STDBUF_I
+GNU-compatible variable for
+.Ev STDBUF_0
+.It Ev _STDBUF_O
+GNU-compatible variable equivalent to
+.Ev STDBUF_1
+.It Ev _STDBUF_E
+GNU-compatible variable equivalent to
+.Ev STDBUF_2
+.El
+.Pp
+Each variable may take one of the following values:
+.Bl -tag -width size -offset indent
+.It Qq 0
+unbuffered
+.It Qq L
+line buffered
+.It Qq B
+fully buffered with the default buffer size
+.It Ar size
+fully buffered with a buffer of
+.Ar size
+bytes
+.El
+.Sh EXAMPLE
+In the following example, the stdout stream of the
+.Xr awk 1
+command
+will be fully buffered by default because it does not refer
+to a terminal.
+.Nm
+is used to force it to be line-buffered so
+.Xr vmstat 8 Ns 's
+output will not stall until the full buffer fills.
+.Bd -literal -offset indent
+# vmstat 1 | LD_PRELOAD=/usr/lib/libstdbuf.so \\
+ STDBUF_1=L awk '$2 > 1 || $3 > 1' | cat -n
+.Ed
+.Pp
+See also the manpage of
+.Xr stdbuf 1
+for a simpler way to do this.
+.Sh HISTORY
+The
+.Nm
+library first appeared in
+.Fx 9.0 .
+.Sh AUTHORS
+.An -nosplit
+The original idea of the
+.Nm
+command comes from
+.An Padraig Brady
+who implemented it in the GNU coreutils.
+.An Jeremie Le Hen
+implemented it on
+.Fx .
diff -urNp src.HEAD_20111506/lib/libstdbuf/stdbuf.c src/lib/libstdbuf/stdbuf.c
--- src.HEAD_20111506/lib/libstdbuf/stdbuf.c 1970-01-01 01:00:00.000000000 +0100
+++ src/lib/libstdbuf/stdbuf.c 2011-08-04 18:17:07.000000000 +0200
@@ -0,0 +1,105 @@
+/*-
+ * Copyright (c) 2011 Jeremie Le Hen
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <err.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <libutil.h>
+
+static const char *
+stream_name(FILE *s)
+{
+ if (s == stdin)
+ return "stdin";
+ if (s == stdout)
+ return "stdout";
+ if (s == stderr)
+ return "stderr";
+ /* This should not happen. */
+ abort();
+}
+
+static void
+change_buf(FILE *s, const char *bufmode)
+{
+ size_t bufsiz;
+ uint64_t sizearg;
+ int mode;
+
+ bufsiz = 0;
+ if (bufmode[0] == '0' && bufmode[1] == '\0')
+ mode = _IONBF;
+ else if (bufmode[0] == 'L' && bufmode[1] == '\0')
+ mode = _IOLBF;
+ else if (bufmode[0] == 'B' && bufmode[1] == '\0') {
+ mode = _IOFBF;
+ bufsiz = 0;
+ } else {
+ errno = 0;
+ if (expand_number(bufmode, &sizearg) == -1) {
+ warn("Wrong buffer mode '%s' for %s", bufmode,
+ stream_name(s));
+ return;
+ }
+ if (sizearg > SIZE_T_MAX) {
+ warn("Buffer size too big for %s", stream_name(s));
+ return;
+ }
+ mode = _IOFBF;
+ bufsiz = (size_t)sizearg;
+ }
+ if (setvbuf(s, NULL, mode, bufsiz) != 0)
+ warn("Cannot set buffer mode '%s' for %s", bufmode,
+ stream_name(s));
+}
+
+__attribute__ ((constructor)) static void
+stdbuf(void)
+{
+ char *i_mode, *o_mode, *e_mode;
+
+ i_mode = getenv("STDBUF_0");
+ if (i_mode == NULL)
+ i_mode = getenv("_STDBUF_I");
+ o_mode = getenv("STDBUF_1");
+ if (o_mode == NULL)
+ o_mode = getenv("_STDBUF_O");
+ e_mode = getenv("STDBUF_2");
+ if (e_mode == NULL)
+ e_mode = getenv("_STDBUF_E");
+
+ if (e_mode)
+ change_buf(stderr, e_mode);
+ if (i_mode)
+ change_buf(stdin, i_mode);
+ if (o_mode)
+ change_buf(stdout, o_mode);
+}
diff -urNp src.HEAD_20111506/usr.bin/stdbuf/Makefile src/usr.bin/stdbuf/Makefile
--- src.HEAD_20111506/usr.bin/stdbuf/Makefile 1970-01-01 01:00:00.000000000 +0100
+++ src/usr.bin/stdbuf/Makefile 2011-08-02 19:16:30.000000000 +0200
@@ -0,0 +1,8 @@
+# $Id$
+
+PROG= stdbuf
+SRCS= stdbuf.c
+
+WARNS?= 6
+
+.include <bsd.prog.mk>
Files src.HEAD_20111506/usr.bin/stdbuf/sh.core and src/usr.bin/stdbuf/sh.core differ
diff -urNp src.HEAD_20111506/usr.bin/stdbuf/stdbuf.1 src/usr.bin/stdbuf/stdbuf.1
--- src.HEAD_20111506/usr.bin/stdbuf/stdbuf.1 1970-01-01 01:00:00.000000000 +0100
+++ src/usr.bin/stdbuf/stdbuf.1 2011-08-04 18:46:35.000000000 +0200
@@ -0,0 +1,124 @@
+.\" Copyright (C) 2011 Jeremie Le Hen
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code and documentation must retain the above
+.\" copyright notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\" notice, this list of conditions and the following disclaimer in the
+.\" documentation and/or other materials provided with the distribution.
+.\" 3. All advertising materials mentioning features or use of this software
+.\" must display the following acknowledgement:
+.\" This product includes software developed or owned by Caldera
+.\" International, Inc.
+.\" 4. Neither the name of Caldera International, Inc. nor the names of other
+.\" contributors may be used to endorse or promote products derived from
+.\" this software without specific prior written permission.
+.\"
+.\" USE OF THE SOFTWARE PROVIDED FOR UNDER THIS LICENSE BY CALDERA
+.\" INTERNATIONAL, INC. AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR
+.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+.\" IN NO EVENT SHALL CALDERA INTERNATIONAL, INC. BE LIABLE FOR ANY DIRECT,
+.\" INDIRECT INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+.\" (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+.\" SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" $FreeBSD: src/usr.bin/dc/dc.1,v 1.2 2010/01/22 23:50:46 delphij Exp $
+.\"
+.Dd August 4, 2011
+.Dt STDBUF 1
+.Os
+.Sh NAME
+.Nm stdbuf
+.Nd change standard streams initial buffering
+.Sh SYNOPSIS
+.Nm
+.Op Fl e Ar bufdef
+.Op Fl i Ar bufdef
+.Op Fl o Ar bufdef
+.Op Ar command Op ...
+.Sh DESCRIPTION
+.Nm
+is used to change the initial buffering of standard input,
+standard output and/or standard error streams for
+.Ar command .
+It relies on
+.Xr libstdbuf 3
+which is loaded and configured by
+.Nm
+through environment variables.
+.Pp
+The options are as follows:
+.Bl -tag -width Ds
+.It Fl e Ar bufdef
+Set initial buffering of the standard error stream for
+.Ar command
+as defined by
+.Ar bufdef
+.Pq see Sx BUFFER DEFINITION .
+.It Fl i Ar bufdef
+Set initial buffering of the standard input stream for
+.Ar command
+as defined by
+.Ar bufdef
+.Pq see Sx BUFFER DEFINITION .
+.It Fl o Ar bufdef
+Set initial buffering of the standard output stream for
+.Ar command
+as defined by
+.Ar bufdef
+.Pq see Sx BUFFER DEFINITION .
+.El
+.Sh BUFFER DEFINITION
+Buffer definition is the same as in
+.Xr libstdbuf 3 :
+.Bl -tag -width size -offset indent
+.It Qq 0
+unbuffered
+.It Qq L
+line buffered
+.It Qq B
+fully buffered with the default buffer size
+.It Ar size
+fully buffered with a buffer of
+.Ar size
+bytes
+.El
+.Sh EXAMPLES
+In the following example, the stdout stream of the
+.Xr awk 1
+command
+will be fully buffered by default because it does not refer
+to a terminal.
+.Nm
+is used to force it to be line-buffered so
+.Xr vmstat 8 Ns 's
+output will not stall until the full buffer fills.
+.Bd -literal -offset indent
+# vmstat 1 | stdbuf -o L awk '$2 > 1 || $3 > 1' | cat -n
+.Ed
+.Sh SEE ALSO
+.Xr libstdbuf 3 ,
+.Xr setvbuf 3
+.Sh HISTORY
+The
+.Nm
+utility first appeared in
+.Fx 9.0 .
+.Sh AUTHORS
+.An -nosplit
+The original idea of the
+.Nm
+command comes from
+.An Padraig Brady
+who implemented it in the GNU coreutils.
+.An Jeremie Le Hen
+implemented it on
+.Fx .
diff -urNp src.HEAD_20111506/usr.bin/stdbuf/stdbuf.c src/usr.bin/stdbuf/stdbuf.c
--- src.HEAD_20111506/usr.bin/stdbuf/stdbuf.c 1970-01-01 01:00:00.000000000 +0100
+++ src/usr.bin/stdbuf/stdbuf.c 2011-08-04 18:17:50.000000000 +0200
@@ -0,0 +1,107 @@
+/*-
+ * Copyright (c) 2011 Jeremie Le Hen
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#define LIBSTDBUF "/usr/lib/libstdbuf.so"
+
+extern char *__progname;
+
+static void usage(int);
+
+static void
+usage(int s)
+{
+
+ fprintf(stderr, "Usage: %s [-e bufdef] [-i bufdef] [-o bufdef] "
+ "<command> [args ...]\n", __progname);
+ fprintf(stderr, " ``bufdef'' being:\n");
+ fprintf(stderr, " - \"0\" to disable buffering;\n");
+ fprintf(stderr, " - \"L\" to set line-buffering;\n");
+ fprintf(stderr, " - <size> to set full-buffering.\n");
+ exit(s);
+}
+
+int
+main(int argc, char *argv[])
+{
+ int i;
+ char *ibuf = NULL, *obuf = NULL, *ebuf = NULL;
+ char *preload0, *preload1;
+
+ while ((i = getopt(argc, argv, ":e:i:o:")) != -1) {
+ switch (i) {
+ case 'e':
+ ebuf = optarg;
+ break;
+ case 'i':
+ ibuf = optarg;
+ break;
+ case 'o':
+ obuf = optarg;
+ break;
+ case ':':
+ warnx("Missing argument for option -%c", optopt);
+ usage(1);
+ break;
+ case '?':
+ default:
+ warnx("Unknown option -%c", optopt);
+ usage(1);
+ break;
+ }
+ }
+ argc -= optind;
+ argv += optind;
+
+ if (ibuf != NULL && setenv("STDBUF_0", ibuf, 1) == -1)
+ warn("Failed to set environment variable: %s=%s",
+ "STDBUF_0", ibuf);
+ if (obuf != NULL && setenv("STDBUF_1", obuf, 1) == -1)
+ warn("Failed to set environment variable: %s=%s",
+ "STDBUF_1", obuf);
+ if (ebuf != NULL && setenv("STDBUF_2", ebuf, 1) == -1)
+ warn("Failed to set environment variable: %s=%s",
+ "STDBUF_2", ebuf);
+
+ preload0 = getenv("LD_PRELOAD");
+ if (preload0 == NULL)
+ i = asprintf(&preload1, "LD_PRELOAD=" LIBSTDBUF);
+ else
+ i = asprintf(&preload1, "LD_PRELOAD=%s:%s", preload0,
+ LIBSTDBUF);
+
+ if (i < 0 || putenv(preload1) == -1)
+ warn("Failed to set environment variable: %s", preload1);
+
+ execvp(argv[0], argv);
+ err(2, "%s", argv[0]);
+}
--- stdbuf.diff ends here ---
>Release-Note:
>Audit-Trail:
>Unformatted:
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?20120405072458.6E55DD95F>