Date: Sun, 8 Dec 2019 04:19:06 +0000 (UTC) From: Matt Macy <mmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r355520 - stable/12/bin/dd Message-ID: <201912080419.xB84J6Dc011288@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: mmacy Date: Sun Dec 8 04:19:05 2019 New Revision: 355520 URL: https://svnweb.freebsd.org/changeset/base/355520 Log: MFC r351770,r352920-r352923 MFCs to dd appear to have been haphazard so selectively MFCing individually didn't work easily. Add conv=fsync flag to dd The fsync flag performs an fsync(2) on the output file before closing it. This will be useful for the ZFS test suite. Add conv=fdatasync flag to dd The fdatasync flag performs an fdatasync(2) on the output file before closing it. This will be useful for the ZFS test suite. dd: Check result of close(2) for errors close(2) can return errors from previous operations which should not be ignored. Add oflag=fsync and oflag=sync capability to dd Sets the O_FSYNC flag on the output file. oflag=fsync and oflag=sync are synonyms just as O_FSYNC and O_SYNC are synonyms. This functionality is intended to improve portability of dd commands in the ZFS test suite. Add iflag=fullblock to dd Normally, count=n means read(2) will be called n times on the input to dd. If the read() returns short, as may happen when reading from a pipe, fewer bytes will be copied from the input. With conv=sync the buffer is padded with zeros to fill the rest of the block. iflag=fullblock causes dd to continue reading until the block is full, so that count=n means n full blocks are copied. This flag is compatible with illumos and GNU dd and is used in the ZFS test suite. Submitted by: Ryan Moeller, Thomas Hurst Reviewed by: manpages, mmacy@ Sponsored by: iXsystems, Inc. Modified: stable/12/bin/dd/args.c stable/12/bin/dd/dd.1 stable/12/bin/dd/dd.c stable/12/bin/dd/dd.h stable/12/bin/dd/extern.h Modified: stable/12/bin/dd/args.c ============================================================================== --- stable/12/bin/dd/args.c Sun Dec 8 04:17:04 2019 (r355519) +++ stable/12/bin/dd/args.c Sun Dec 8 04:19:05 2019 (r355520) @@ -41,7 +41,7 @@ static char sccsid[] = "@(#)args.c 8.3 (Berkeley) 4/2/ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <sys/types.h> +#include <sys/param.h> #include <ctype.h> #include <err.h> @@ -57,6 +57,8 @@ __FBSDID("$FreeBSD$"); static int c_arg(const void *, const void *); static int c_conv(const void *, const void *); +static int c_iflag(const void *, const void *); +static int c_oflag(const void *, const void *); static void f_bs(char *); static void f_cbs(char *); static void f_conv(char *); @@ -65,8 +67,10 @@ static void f_files(char *); static void f_fillchar(char *); static void f_ibs(char *); static void f_if(char *); +static void f_iflag(char *); static void f_obs(char *); static void f_of(char *); +static void f_oflag(char *); static void f_seek(char *); static void f_skip(char *); static void f_speed(char *); @@ -77,7 +81,7 @@ static off_t get_off_t(const char *); static const struct arg { const char *name; void (*f)(char *); - u_int set, noset; + uint64_t set, noset; } args[] = { { "bs", f_bs, C_BS, C_BS|C_IBS|C_OBS|C_OSYNC }, { "cbs", f_cbs, C_CBS, C_CBS }, @@ -87,9 +91,11 @@ static const struct arg { { "fillchar", f_fillchar, C_FILL, C_FILL }, { "ibs", f_ibs, C_IBS, C_BS|C_IBS }, { "if", f_if, C_IF, C_IF }, + { "iflag", f_iflag, 0, 0 }, { "iseek", f_skip, C_SKIP, C_SKIP }, { "obs", f_obs, C_OBS, C_BS|C_OBS }, { "of", f_of, C_OF, C_OF }, + { "oflag", f_oflag, 0, 0 }, { "oseek", f_seek, C_SEEK, C_SEEK }, { "seek", f_seek, C_SEEK, C_SEEK }, { "skip", f_skip, C_SKIP, C_SKIP }, @@ -256,7 +262,39 @@ f_if(char *arg) in.name = arg; } +static const struct iflag { + const char *name; + uint64_t set, noset; +} ilist[] = { + { "fullblock", C_IFULLBLOCK, C_SYNC }, +}; + static void +f_iflag(char *arg) +{ + struct iflag *ip, tmp; + + while (arg != NULL) { + tmp.name = strsep(&arg, ","); + ip = bsearch(&tmp, ilist, nitems(ilist), sizeof(struct iflag), + c_iflag); + if (ip == NULL) + errx(1, "unknown iflag %s", tmp.name); + if (ddflags & ip->noset) + errx(1, "%s: illegal conversion combination", tmp.name); + ddflags |= ip->set; + } +} + +static int +c_iflag(const void *a, const void *b) +{ + + return (strcmp(((const struct iflag *)a)->name, + ((const struct iflag *)b)->name)); +} + +static void f_obs(char *arg) { uintmax_t res; @@ -314,12 +352,14 @@ f_status(char *arg) static const struct conv { const char *name; - u_int set, noset; + uint64_t set, noset; const u_char *ctab; } clist[] = { { "ascii", C_ASCII, C_EBCDIC, e2a_POSIX }, { "block", C_BLOCK, C_UNBLOCK, NULL }, { "ebcdic", C_EBCDIC, C_ASCII, a2e_POSIX }, + { "fdatasync", C_FDATASYNC, 0, NULL }, + { "fsync", C_FSYNC, 0, NULL }, { "ibm", C_EBCDIC, C_ASCII, a2ibm_POSIX }, { "lcase", C_LCASE, C_UCASE, NULL }, { "noerror", C_NOERROR, 0, NULL }, @@ -334,7 +374,7 @@ static const struct conv { { "parset", C_PARSET, C_PARODD|C_PAREVEN|C_PARNONE, NULL}, { "sparse", C_SPARSE, 0, NULL }, { "swab", C_SWAB, 0, NULL }, - { "sync", C_SYNC, 0, NULL }, + { "sync", C_SYNC, C_IFULLBLOCK, NULL }, { "ucase", C_UCASE, C_LCASE, NULL }, { "unblock", C_UNBLOCK, C_BLOCK, NULL }, }; @@ -346,8 +386,8 @@ f_conv(char *arg) while (arg != NULL) { tmp.name = strsep(&arg, ","); - cp = bsearch(&tmp, clist, sizeof(clist) / sizeof(struct conv), - sizeof(struct conv), c_conv); + cp = bsearch(&tmp, clist, nitems(clist), sizeof(struct conv), + c_conv); if (cp == NULL) errx(1, "unknown conversion %s", tmp.name); if (ddflags & cp->noset) @@ -364,6 +404,37 @@ c_conv(const void *a, const void *b) return (strcmp(((const struct conv *)a)->name, ((const struct conv *)b)->name)); +} + +static const struct oflag { + const char *name; + uint64_t set; +} olist[] = { + { "fsync", C_OFSYNC }, + { "sync", C_OFSYNC }, +}; + +static void +f_oflag(char *arg) +{ + struct oflag *op, tmp; + + while (arg != NULL) { + tmp.name = strsep(&arg, ","); + op = bsearch(&tmp, olist, nitems(olist), sizeof(struct oflag), + c_oflag); + if (op == NULL) + errx(1, "unknown open flag %s", tmp.name); + ddflags |= op->set; + } +} + +static int +c_oflag(const void *a, const void *b) +{ + + return (strcmp(((const struct oflag *)a)->name, + ((const struct oflag *)b)->name)); } static intmax_t Modified: stable/12/bin/dd/dd.1 ============================================================================== --- stable/12/bin/dd/dd.1 Sun Dec 8 04:17:04 2019 (r355519) +++ stable/12/bin/dd/dd.1 Sun Dec 8 04:19:05 2019 (r355520) @@ -32,7 +32,7 @@ .\" @(#)dd.1 8.2 (Berkeley) 1/13/94 .\" $FreeBSD$ .\" -.Dd August 8, 2018 +.Dd March 26, 2019 .Dt DD 1 .Os .Sh NAME @@ -102,6 +102,22 @@ bytes instead of the default 512. Read input from .Ar file instead of the standard input. +.It Cm iflag Ns = Ns Ar value Ns Op , Ns Ar value ... +Where +.Cm value +is one of the symbols from the following list. +.Bl -tag -width "fullblock" +.It Cm fullblock +Reading from the input file may not obtain a full block. +When a read returns short, continue reading to fill the block. +Without this flag, +.Cm count +limits the number of times +.Xr read 2 +is called on the input rather than the number of blocks copied in full. +May not be combined with +.Cm conv=sync . +.El .It Cm iseek Ns = Ns Ar n Seek on the input file .Ar n @@ -123,6 +139,19 @@ If an initial portion of the output file is seeked pas .Cm oseek operand), the output file is truncated at that point. +.It Cm oflag Ns = Ns Ar value Ns Op , Ns Ar value ... +Where +.Cm value +is one of the symbols from the following list. +.Bl -tag -width "fsync" +.It Cm fsync +Set the O_FSYNC flag on the output file to make writes synchronous. +.It Cm sync +Set the O_SYNC flag on the output file to make writes synchronous. +This is synonymous with the +.Cm fsync +value. +.El .It Cm oseek Ns = Ns Ar n Seek on the output file .Ar n @@ -252,6 +281,14 @@ are maps used in historic and .No pre- Ns Bx 4.3 reno systems. +.It Cm fdatasync +Perform an +.Xr fdatasync 2 +on the output file before closing it. +.It Cm fsync +Perform an +.Xr fsync 2 +on the output file before closing it. .It Cm lcase Transform uppercase characters into lowercase characters. .It Cm pareven , parnone , parodd , parset @@ -427,7 +464,8 @@ if necessary, to a 1MiB boundary: .Xr mt 1 , .Xr recoverdisk 1 , .Xr tr 1 , -.Xr geom 4 +.Xr geom 4 , +.Xr trim 8 .Sh STANDARDS The .Nm Modified: stable/12/bin/dd/dd.c ============================================================================== --- stable/12/bin/dd/dd.c Sun Dec 8 04:17:04 2019 (r355519) +++ stable/12/bin/dd/dd.c Sun Dec 8 04:19:05 2019 (r355520) @@ -83,7 +83,7 @@ STAT st; /* statistics */ void (*cfunc)(void); /* conversion function */ uintmax_t cpy_cnt; /* # of blocks to copy */ static off_t pending = 0; /* pending seek if sparse */ -u_int ddflags = 0; /* conversion options */ +uint64_t ddflags = 0; /* conversion options */ size_t cbsz; /* conversion block size */ uintmax_t files_cnt = 1; /* # of files to copy */ const u_char *ctab; /* conversion table */ @@ -124,7 +124,8 @@ main(int argc __unused, char *argv[]) * descriptor explicitly so that the summary handler (called * from an atexit() hook) includes this work. */ - close(out.fd); + if (close(out.fd) == -1 && errno != EINTR) + err(1, "close"); exit(0); } @@ -142,6 +143,7 @@ static void setup(void) { u_int cnt; + int oflags; cap_rights_t rights; unsigned long cmds[] = { FIODTYPE, MTIOCTOP }; @@ -164,21 +166,34 @@ setup(void) errx(1, "files is not supported for non-tape devices"); cap_rights_set(&rights, CAP_FTRUNCATE, CAP_IOCTL, CAP_WRITE); + if (ddflags & (C_FDATASYNC | C_FSYNC)) + cap_rights_set(&rights, CAP_FSYNC); if (out.name == NULL) { /* No way to check for read access here. */ out.fd = STDOUT_FILENO; out.name = "stdout"; + if (ddflags & C_OFSYNC) { + oflags = fcntl(out.fd, F_GETFL); + if (oflags == -1) + err(1, "unable to get fd flags for stdout"); + oflags |= O_FSYNC; + if (fcntl(out.fd, F_SETFL, oflags) == -1) + err(1, "unable to set fd flags for stdout"); + } } else { -#define OFLAGS \ - (O_CREAT | (ddflags & (C_SEEK | C_NOTRUNC) ? 0 : O_TRUNC)) - out.fd = open(out.name, O_RDWR | OFLAGS, DEFFILEMODE); + oflags = O_CREAT; + if (!(ddflags & (C_SEEK | C_NOTRUNC))) + oflags |= O_TRUNC; + if (ddflags & C_OFSYNC) + oflags |= O_FSYNC; + out.fd = open(out.name, O_RDWR | oflags, DEFFILEMODE); /* * May not have read access, so try again with write only. * Without read we may have a problem if output also does * not support seeks. */ if (out.fd == -1) { - out.fd = open(out.name, O_WRONLY | OFLAGS, DEFFILEMODE); + out.fd = open(out.name, O_WRONLY | oflags, DEFFILEMODE); out.flags |= NOREAD; cap_rights_clear(&rights, CAP_READ); } @@ -393,13 +408,15 @@ dd_in(void) memset(in.dbp, 0, in.dbsz); } - n = read(in.fd, in.dbp, in.dbsz); - if (n == 0) { - in.dbrcnt = 0; + in.dbrcnt = 0; +fill: + n = read(in.fd, in.dbp + in.dbrcnt, in.dbsz - in.dbrcnt); + + /* EOF */ + if (n == 0 && in.dbrcnt == 0) return; - } - /* Read error. */ + /* Read error */ if (n == -1) { /* * If noerror not specified, die. POSIX requires that @@ -423,26 +440,26 @@ dd_in(void) /* If sync not specified, omit block and continue. */ if (!(ddflags & C_SYNC)) continue; + } - /* Read errors count as full blocks. */ - in.dbcnt += in.dbrcnt = in.dbsz; - ++st.in_full; + /* If conv=sync, use the entire block. */ + if (ddflags & C_SYNC) + n = in.dbsz; - /* Handle full input blocks. */ - } else if ((size_t)n == (size_t)in.dbsz) { - in.dbcnt += in.dbrcnt = n; - ++st.in_full; + /* Count the bytes read for this block. */ + in.dbrcnt += n; - /* Handle partial input blocks. */ - } else { - /* If sync, use the entire block. */ - if (ddflags & C_SYNC) - in.dbcnt += in.dbrcnt = in.dbsz; - else - in.dbcnt += in.dbrcnt = n; + /* Count the number of full and partial blocks. */ + if (in.dbrcnt == in.dbsz) + ++st.in_full; + else if (ddflags & C_IFULLBLOCK && n != 0) + goto fill; /* these don't count */ + else ++st.in_part; - } + /* Count the total bytes read for this file. */ + in.dbcnt += in.dbrcnt; + /* * POSIX states that if bs is set and no other conversions * than noerror, notrunc or sync are specified, the block @@ -463,6 +480,7 @@ dd_in(void) swapbytes(in.dbp, (size_t)n); } + /* Advance to the next block. */ in.dbp += in.dbrcnt; (*cfunc)(); if (need_summary) @@ -504,6 +522,14 @@ dd_close(void) if (out.seek_offset > 0 && (out.flags & ISTRUNC)) { if (ftruncate(out.fd, out.seek_offset) == -1) err(1, "truncating %s", out.name); + } + + if (ddflags & C_FSYNC) { + if (fsync(out.fd) == -1) + err(1, "fsyncing %s", out.name); + } else if (ddflags & C_FDATASYNC) { + if (fdatasync(out.fd) == -1) + err(1, "fdatasyncing %s", out.name); } } Modified: stable/12/bin/dd/dd.h ============================================================================== --- stable/12/bin/dd/dd.h Sun Dec 8 04:17:04 2019 (r355519) +++ stable/12/bin/dd/dd.h Sun Dec 8 04:19:05 2019 (r355520) @@ -70,37 +70,41 @@ typedef struct { } STAT; /* Flags (in ddflags). */ -#define C_ASCII 0x00000001 -#define C_BLOCK 0x00000002 -#define C_BS 0x00000004 -#define C_CBS 0x00000008 -#define C_COUNT 0x00000010 -#define C_EBCDIC 0x00000020 -#define C_FILES 0x00000040 -#define C_IBS 0x00000080 -#define C_IF 0x00000100 -#define C_LCASE 0x00000200 -#define C_NOERROR 0x00000400 -#define C_NOTRUNC 0x00000800 -#define C_OBS 0x00001000 -#define C_OF 0x00002000 -#define C_OSYNC 0x00004000 -#define C_PAREVEN 0x00008000 -#define C_PARNONE 0x00010000 -#define C_PARODD 0x00020000 -#define C_PARSET 0x00040000 -#define C_SEEK 0x00080000 -#define C_SKIP 0x00100000 -#define C_SPARSE 0x00200000 -#define C_SWAB 0x00400000 -#define C_SYNC 0x00800000 -#define C_UCASE 0x01000000 -#define C_UNBLOCK 0x02000000 -#define C_FILL 0x04000000 -#define C_STATUS 0x08000000 -#define C_NOXFER 0x10000000 -#define C_NOINFO 0x20000000 -#define C_PROGRESS 0x40000000 +#define C_ASCII 0x0000000000000001ULL +#define C_BLOCK 0x0000000000000002ULL +#define C_BS 0x0000000000000004ULL +#define C_CBS 0x0000000000000008ULL +#define C_COUNT 0x0000000000000010ULL +#define C_EBCDIC 0x0000000000000020ULL +#define C_FILES 0x0000000000000040ULL +#define C_IBS 0x0000000000000080ULL +#define C_IF 0x0000000000000100ULL +#define C_LCASE 0x0000000000000200ULL +#define C_NOERROR 0x0000000000000400ULL +#define C_NOTRUNC 0x0000000000000800ULL +#define C_OBS 0x0000000000001000ULL +#define C_OF 0x0000000000002000ULL +#define C_OSYNC 0x0000000000004000ULL +#define C_PAREVEN 0x0000000000008000ULL +#define C_PARNONE 0x0000000000010000ULL +#define C_PARODD 0x0000000000020000ULL +#define C_PARSET 0x0000000000040000ULL +#define C_SEEK 0x0000000000080000ULL +#define C_SKIP 0x0000000000100000ULL +#define C_SPARSE 0x0000000000200000ULL +#define C_SWAB 0x0000000000400000ULL +#define C_SYNC 0x0000000000800000ULL +#define C_UCASE 0x0000000001000000ULL +#define C_UNBLOCK 0x0000000002000000ULL +#define C_FILL 0x0000000004000000ULL +#define C_STATUS 0x0000000008000000ULL +#define C_NOXFER 0x0000000010000000ULL +#define C_NOINFO 0x0000000020000000ULL +#define C_PROGRESS 0x0000000040000000ULL +#define C_FSYNC 0x0000000080000000ULL +#define C_FDATASYNC 0x0000000100000000ULL +#define C_OFSYNC 0x0000000200000000ULL +#define C_IFULLBLOCK 0x0000000400000000ULL #define C_PARITY (C_PAREVEN | C_PARODD | C_PARNONE | C_PARSET) Modified: stable/12/bin/dd/extern.h ============================================================================== --- stable/12/bin/dd/extern.h Sun Dec 8 04:17:04 2019 (r355519) +++ stable/12/bin/dd/extern.h Sun Dec 8 04:19:05 2019 (r355520) @@ -58,7 +58,7 @@ extern STAT st; extern void (*cfunc)(void); extern uintmax_t cpy_cnt; extern size_t cbsz; -extern u_int ddflags; +extern uint64_t ddflags; extern size_t speed; extern uintmax_t files_cnt; extern const u_char *ctab;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201912080419.xB84J6Dc011288>