Date: Fri, 4 Nov 2022 13:01:33 GMT From: =?utf-8?Q?Dag-Erling=20Sm=C3=B8rgrav?= <des@FreeBSD.org> To: src-committers@FreeBSD.org, dev-commits-src-all@FreeBSD.org, dev-commits-src-main@FreeBSD.org Subject: git: f9349d427431 - main - gzip: Add support for decompressing zstd files. Message-ID: <202211041301.2A4D1X6t096180@gitrepo.freebsd.org>
next in thread | raw e-mail | index | archive | help
The branch main has been updated by des: URL: https://cgit.FreeBSD.org/src/commit/?id=f9349d427431eaa5b3bd7e9e7e87725a138facd1 commit f9349d427431eaa5b3bd7e9e7e87725a138facd1 Author: Dag-Erling Smørgrav <des@FreeBSD.org> AuthorDate: 2022-11-02 13:41:44 +0000 Commit: Dag-Erling Smørgrav <des@FreeBSD.org> CommitDate: 2022-11-04 12:42:09 +0000 gzip: Add support for decompressing zstd files. Sponsored by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D37236 --- usr.bin/gzip/Makefile | 4 +- usr.bin/gzip/Makefile.depend | 1 + usr.bin/gzip/gzip.1 | 4 +- usr.bin/gzip/gzip.c | 60 ++++++++++++++++++++++------- usr.bin/gzip/unzstd.c | 89 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 15 deletions(-) diff --git a/usr.bin/gzip/Makefile b/usr.bin/gzip/Makefile index c8ec4743f98d..227dce362a45 100644 --- a/usr.bin/gzip/Makefile +++ b/usr.bin/gzip/Makefile @@ -6,7 +6,7 @@ PROG= gzip MAN= gzip.1 gzexe.1 zdiff.1 zforce.1 zmore.1 znew.1 -LIBADD= z lzma +LIBADD= z lzma zstd .if ${MK_BZIP2_SUPPORT} != "no" LIBADD+= bz2 @@ -14,6 +14,8 @@ LIBADD+= bz2 CFLAGS+= -DNO_BZIP2_SUPPORT .endif +CFLAGS+= -I${SRCTOP}/sys/contrib/zstd/lib + SCRIPTS= gzexe zdiff zforce zmore znew MLINKS+= gzip.1 gunzip.1 \ diff --git a/usr.bin/gzip/Makefile.depend b/usr.bin/gzip/Makefile.depend index 02d864779e18..9a4b70335fec 100644 --- a/usr.bin/gzip/Makefile.depend +++ b/usr.bin/gzip/Makefile.depend @@ -10,6 +10,7 @@ DIRDEPS = \ lib/liblzma \ lib/libthr \ lib/libz \ + lib/libzstd \ .include <dirdeps.mk> diff --git a/usr.bin/gzip/gzip.1 b/usr.bin/gzip/gzip.1 index a8293b748832..356568a2ba11 100644 --- a/usr.bin/gzip/gzip.1 +++ b/usr.bin/gzip/gzip.1 @@ -25,7 +25,7 @@ .\" SUCH DAMAGE. .\" .\" $FreeBSD$ -.Dd January 7, 2019 +.Dd November 2, 2022 .Dt GZIP 1 .Os .Sh NAME @@ -110,6 +110,7 @@ is also capable of decompressing files compressed using .Xr compress 1 , .Xr bzip2 1 , .Ar lzip , +.Xr zstd 1 , or .Xr xz 1 . .Sh OPTIONS @@ -203,6 +204,7 @@ Report progress to standard error. .Sh SEE ALSO .Xr bzip2 1 , .Xr compress 1 , +.Xr zstd 1 , .Xr xz 1 , .Xr fts 3 , .Xr zlib 3 diff --git a/usr.bin/gzip/gzip.c b/usr.bin/gzip/gzip.c index bdafb9e0b457..a7f64b61d053 100644 --- a/usr.bin/gzip/gzip.c +++ b/usr.bin/gzip/gzip.c @@ -86,6 +86,9 @@ enum filetype { #endif #ifndef NO_LZ_SUPPORT FT_LZ, +#endif +#ifndef NO_ZSTD_SUPPORT + FT_ZSTD, #endif FT_LAST, FT_UNKNOWN @@ -118,6 +121,12 @@ enum filetype { #define LZ_MAGIC "LZIP" #endif +#ifndef NO_ZSTD_SUPPORT +#include <zstd.h> +#define ZSTD_SUFFIX ".zst" +#define ZSTD_MAGIC "\050\265\057\375" +#endif + #define GZ_SUFFIX ".gz" #define BUFLEN (64 * 1024) @@ -164,6 +173,9 @@ static suffixes_t suffixes[] = { #endif #ifndef NO_LZ_SUPPORT SUFFIX(LZ_SUFFIX, ""), +#endif +#ifndef NO_ZSTD_SUPPORT + SUFFIX(ZSTD_SUFFIX, ""), #endif SUFFIX(GZ_SUFFIX, ""), /* Overwritten by -S "" */ #undef SUFFIX @@ -221,7 +233,7 @@ static const char *infile; /* name of file coming in */ static void maybe_err(const char *fmt, ...) __printflike(1, 2) __dead2; #if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) || \ - !defined(NO_XZ_SUPPORT) + !defined(NO_XZ_SUPPORT) || !defined(NO_ZSTD_SUPPORT) static void maybe_errx(const char *fmt, ...) __printflike(1, 2) __dead2; #endif static void maybe_warn(const char *fmt, ...) __printflike(1, 2); @@ -284,6 +296,10 @@ static off_t unxz_len(int); static off_t unlz(int, int, char *, size_t, off_t *); #endif +#ifndef NO_ZSTD_SUPPORT +static off_t unzstd(int, int, char *, size_t, off_t *); +#endif + static const struct option longopts[] = { { "stdout", no_argument, 0, 'c' }, { "to-stdout", no_argument, 0, 'c' }, @@ -467,7 +483,7 @@ maybe_err(const char *fmt, ...) } #if !defined(NO_BZIP2_SUPPORT) || !defined(NO_PACK_SUPPORT) || \ - !defined(NO_XZ_SUPPORT) + !defined(NO_XZ_SUPPORT) || !defined(NO_ZSTD_SUPPORT) /* ... without an errno. */ void maybe_errx(const char *fmt, ...) @@ -1098,33 +1114,32 @@ file_gettype(u_char *buf) if (buf[0] == GZIP_MAGIC0 && (buf[1] == GZIP_MAGIC1 || buf[1] == GZIP_OMAGIC1)) return FT_GZIP; - else #ifndef NO_BZIP2_SUPPORT - if (memcmp(buf, BZIP2_MAGIC, 3) == 0 && + else if (memcmp(buf, BZIP2_MAGIC, 3) == 0 && buf[3] >= '0' && buf[3] <= '9') return FT_BZIP2; - else #endif #ifndef NO_COMPRESS_SUPPORT - if (memcmp(buf, Z_MAGIC, 2) == 0) + else if (memcmp(buf, Z_MAGIC, 2) == 0) return FT_Z; - else #endif #ifndef NO_PACK_SUPPORT - if (memcmp(buf, PACK_MAGIC, 2) == 0) + else if (memcmp(buf, PACK_MAGIC, 2) == 0) return FT_PACK; - else #endif #ifndef NO_XZ_SUPPORT - if (memcmp(buf, XZ_MAGIC, 4) == 0) /* XXX: We only have 4 bytes */ + else if (memcmp(buf, XZ_MAGIC, 4) == 0) /* XXX: We only have 4 bytes */ return FT_XZ; - else #endif #ifndef NO_LZ_SUPPORT - if (memcmp(buf, LZ_MAGIC, 4) == 0) + else if (memcmp(buf, LZ_MAGIC, 4) == 0) return FT_LZ; - else #endif +#ifndef NO_ZSTD_SUPPORT + else if (memcmp(buf, ZSTD_MAGIC, 4) == 0) + return FT_ZSTD; +#endif + else return FT_UNKNOWN; } @@ -1586,6 +1601,16 @@ file_uncompress(char *file, char *outfile, size_t outsize) size = unlz(fd, zfd, NULL, 0, NULL); break; #endif + +#ifndef NO_ZSTD_SUPPORT + case FT_ZSTD: + if (lflag) { + maybe_warnx("no -l with zstd files"); + goto lose; + } + size = unzstd(fd, zfd, NULL, 0, NULL); + break; +#endif case FT_UNKNOWN: if (lflag) { maybe_warnx("no -l for unknown filetypes"); @@ -1813,6 +1838,12 @@ handle_stdin(void) usize = unlz(STDIN_FILENO, STDOUT_FILENO, (char *)fourbytes, sizeof fourbytes, &gsize); break; +#endif +#ifndef NO_ZSTD_SUPPORT + case FT_ZSTD: + usize = unzstd(STDIN_FILENO, STDOUT_FILENO, + (char *)fourbytes, sizeof fourbytes, &gsize); + break; #endif } @@ -2188,6 +2219,9 @@ display_version(void) #ifndef NO_LZ_SUPPORT #include "unlz.c" #endif +#ifndef NO_ZSTD_SUPPORT +#include "unzstd.c" +#endif static ssize_t read_retry(int fd, void *buf, size_t sz) diff --git a/usr.bin/gzip/unzstd.c b/usr.bin/gzip/unzstd.c new file mode 100644 index 000000000000..4536f3119ace --- /dev/null +++ b/usr.bin/gzip/unzstd.c @@ -0,0 +1,89 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2022 Klara, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* This file is #included by gzip.c */ + +static off_t +unzstd(int in, int out, char *pre, size_t prelen, off_t *bytes_in) +{ + static char *ibuf, *obuf; + ZSTD_inBuffer zib; + ZSTD_outBuffer zob; + ZSTD_DCtx *zds; + ssize_t res; + size_t zres; + size_t bytes_out = 0; + int eof = 0; + + if (ibuf == NULL) + ibuf = malloc(BUFLEN); + if (obuf == NULL) + obuf = malloc(BUFLEN); + if (ibuf == NULL || obuf == NULL) + maybe_err("malloc"); + + zds = ZSTD_createDStream(); + ZSTD_initDStream(zds); + + zib.src = pre; + zib.size = prelen; + zib.pos = 0; + if (bytes_in != NULL) + *bytes_in = prelen; + zob.dst = obuf; + zob.size = BUFLEN; + zob.pos = 0; + + while (!eof) { + if (zib.pos >= zib.size) { + res = read(in, ibuf, BUFLEN); + if (res < 0) + maybe_err("read"); + if (res == 0) + eof = 1; + infile_newdata(res); + zib.src = ibuf; + zib.size = res; + zib.pos = 0; + if (bytes_in != NULL) + *bytes_in += res; + } + zres = ZSTD_decompressStream(zds, &zob, &zib); + if (ZSTD_isError(zres)) { + maybe_errx("%s", ZSTD_getErrorName(zres)); + } + if (zob.pos > 0) { + res = write(out, obuf, zob.pos); + if (res < 0) + maybe_err("write"); + zob.pos = 0; + bytes_out += res; + } + } + ZSTD_freeDStream(zds); + return (bytes_out); +}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202211041301.2A4D1X6t096180>