Date: Thu, 8 Aug 2019 16:54:22 +0000 (UTC) From: Conrad Meyer <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r350760 - in head: lib/libzstd sys/conf sys/contrib/zstd sys/contrib/zstd/contrib/largeNbDicts sys/contrib/zstd/contrib/seekable_format sys/contrib/zstd/contrib/seekable_format/examples... Message-ID: <201908081654.x78GsMU6055994@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Thu Aug 8 16:54:22 2019 New Revision: 350760 URL: https://svnweb.freebsd.org/changeset/base/350760 Log: Update to Zstandard 1.4.2 The full release notes for 1.4.1 (skipped) and 1.4.2 can be found on Github: https://github.com/facebook/zstd/releases/tag/v1.4.1 https://github.com/facebook/zstd/releases/tag/v1.4.2 These are mostly minor updates; 1.4.1 purportedly brings something like 7% faster decompression speed. Relnotes: yes Added: head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c - copied unchanged from r350755, vendor/zstd/dist/contrib/seekable_format/examples/seekable_decompression_mem.c head/sys/contrib/zstd/lib/compress/zstd_compress_literals.c - copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_literals.c head/sys/contrib/zstd/lib/compress/zstd_compress_literals.h - copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_literals.h head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.c - copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_sequences.c head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.h - copied unchanged from r350755, vendor/zstd/dist/lib/compress/zstd_compress_sequences.h head/sys/contrib/zstd/tests/bigdict.c - copied unchanged from r350755, vendor/zstd/dist/tests/bigdict.c head/sys/contrib/zstd/tests/fuzz/simple_compress.c - copied unchanged from r350755, vendor/zstd/dist/tests/fuzz/simple_compress.c head/sys/contrib/zstd/tests/fuzz/zstd_frame_info.c - copied unchanged from r350755, vendor/zstd/dist/tests/fuzz/zstd_frame_info.c Deleted: head/sys/contrib/zstd/tests/fuzz/default.options Modified: head/lib/libzstd/Makefile head/sys/conf/files head/sys/conf/kern.pre.mk head/sys/contrib/zstd/CHANGELOG head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c head/sys/contrib/zstd/doc/zstd_compression_format.md head/sys/contrib/zstd/doc/zstd_manual.html head/sys/contrib/zstd/examples/Makefile head/sys/contrib/zstd/lib/Makefile head/sys/contrib/zstd/lib/common/compiler.h head/sys/contrib/zstd/lib/common/zstd_internal.h head/sys/contrib/zstd/lib/compress/zstd_compress.c head/sys/contrib/zstd/lib/compress/zstd_compress_internal.h head/sys/contrib/zstd/lib/compress/zstd_double_fast.c head/sys/contrib/zstd/lib/compress/zstd_fast.c head/sys/contrib/zstd/lib/compress/zstd_lazy.c head/sys/contrib/zstd/lib/compress/zstd_ldm.c head/sys/contrib/zstd/lib/compress/zstd_opt.c head/sys/contrib/zstd/lib/compress/zstdmt_compress.c head/sys/contrib/zstd/lib/compress/zstdmt_compress.h head/sys/contrib/zstd/lib/decompress/zstd_decompress.c head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.c head/sys/contrib/zstd/lib/dictBuilder/cover.c head/sys/contrib/zstd/lib/dictBuilder/cover.h head/sys/contrib/zstd/lib/dictBuilder/fastcover.c head/sys/contrib/zstd/lib/dictBuilder/zdict.c head/sys/contrib/zstd/lib/dictBuilder/zdict.h head/sys/contrib/zstd/lib/legacy/zstd_legacy.h head/sys/contrib/zstd/lib/legacy/zstd_v01.c head/sys/contrib/zstd/lib/legacy/zstd_v02.c head/sys/contrib/zstd/lib/legacy/zstd_v03.c head/sys/contrib/zstd/lib/legacy/zstd_v04.c head/sys/contrib/zstd/lib/legacy/zstd_v05.c head/sys/contrib/zstd/lib/legacy/zstd_v06.c head/sys/contrib/zstd/lib/legacy/zstd_v07.c head/sys/contrib/zstd/lib/zstd.h head/sys/contrib/zstd/programs/.gitignore head/sys/contrib/zstd/programs/README.md head/sys/contrib/zstd/programs/benchfn.c head/sys/contrib/zstd/programs/fileio.c head/sys/contrib/zstd/programs/fileio.h head/sys/contrib/zstd/programs/util.c head/sys/contrib/zstd/programs/zstd.1 head/sys/contrib/zstd/programs/zstd.1.md head/sys/contrib/zstd/programs/zstdcli.c head/sys/contrib/zstd/programs/zstdgrep head/sys/contrib/zstd/programs/zstdgrep.1 head/sys/contrib/zstd/programs/zstdless.1 head/sys/contrib/zstd/tests/.gitignore head/sys/contrib/zstd/tests/Makefile head/sys/contrib/zstd/tests/decodecorpus.c head/sys/contrib/zstd/tests/fullbench.c head/sys/contrib/zstd/tests/fuzz/Makefile head/sys/contrib/zstd/tests/fuzz/dictionary_decompress.c head/sys/contrib/zstd/tests/fuzz/fuzz.py head/sys/contrib/zstd/tests/fuzz/simple_decompress.c head/sys/contrib/zstd/tests/fuzzer.c head/sys/contrib/zstd/tests/paramgrill.c head/sys/contrib/zstd/tests/playTests.sh head/sys/contrib/zstd/tests/poolTests.c head/sys/contrib/zstd/tests/regression/results.csv head/sys/contrib/zstd/tests/zstreamtest.c head/sys/contrib/zstd/zlibWrapper/.gitignore Directory Properties: head/sys/contrib/zstd/ (props changed) Modified: head/lib/libzstd/Makefile ============================================================================== --- head/lib/libzstd/Makefile Thu Aug 8 16:51:49 2019 (r350759) +++ head/lib/libzstd/Makefile Thu Aug 8 16:54:22 2019 (r350760) @@ -11,6 +11,8 @@ SRCS= entropy_common.c \ fse_compress.c \ huf_compress.c \ zstd_compress.c \ + zstd_compress_literals.c \ + zstd_compress_sequences.c \ zstdmt_compress.c \ huf_decompress.c \ zstd_ddict.c \ @@ -43,7 +45,20 @@ ZSTDDIR= ${SRCTOP}/sys/contrib/zstd ${ZSTDDIR}/lib/decompress ${ZSTDDIR}/lib/deprecated \ ${ZSTDDIR}/lib/dictBuilder ${ZSTDDIR}/lib +.include <bsd.compiler.mk> + +# https://github.com/facebook/zstd/commit/812e8f2a [zstd 1.4.1] +# "Note that [GCC] autovectorization still does not do a good job on the +# optimized version, so it's turned off via attribute and flag. I found +# that neither attribute nor command-line flag were entirely successful in +# turning off vectorization, which is why there were both." +.if ${COMPILER_TYPE} == "gcc" +CFLAGS.zstd_decompress_block.c+= -fno-tree-vectorize +.endif + # Work around for LLVM bug 35023, https://bugs.llvm.org/show_bug.cgi?id=35023 +# Fixed in https://reviews.llvm.org/rL349935 ; not sure if we have that version +# in our LLVM or not yet. .if ${MACHINE_ARCH:Marm*} != "" CFLAGS.zstd_compress.c+= -O0 .endif Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/conf/files Thu Aug 8 16:54:22 2019 (r350760) @@ -648,6 +648,8 @@ contrib/zstd/lib/common/entropy_common.c optional zstd contrib/zstd/lib/common/error_private.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/common/xxhash.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/zstd_compress.c optional zstdio compile-with ${ZSTD_C} +contrib/zstd/lib/compress/zstd_compress_literals.c optional zstdio compile-with ${ZSTD_C} +contrib/zstd/lib/compress/zstd_compress_sequences.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/fse_compress.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/hist.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/huf_compress.c optional zstdio compile-with ${ZSTD_C} @@ -658,7 +660,9 @@ contrib/zstd/lib/compress/zstd_ldm.c optional zstdio contrib/zstd/lib/compress/zstd_opt.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/decompress/zstd_ddict.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/decompress/zstd_decompress.c optional zstdio compile-with ${ZSTD_C} -contrib/zstd/lib/decompress/zstd_decompress_block.c optional zstdio compile-with ${ZSTD_C} +# See comment in sys/conf/kern.pre.mk +contrib/zstd/lib/decompress/zstd_decompress_block.c optional zstdio \ + compile-with "${ZSTD_C} ${ZSTD_DECOMPRESS_BLOCK_FLAGS}" contrib/zstd/lib/decompress/huf_decompress.c optional zstdio compile-with ${ZSTD_C} # Blake 2 contrib/libb2/blake2b-ref.c optional crypto | ipsec | ipsec_support \ Modified: head/sys/conf/kern.pre.mk ============================================================================== --- head/sys/conf/kern.pre.mk Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/conf/kern.pre.mk Thu Aug 8 16:54:22 2019 (r350760) @@ -175,6 +175,14 @@ NORMAL_FWO= ${LD} -b binary --no-warn-mismatch -d -war # for ZSTD in the kernel (include zstd/lib/freebsd before other CFLAGS) ZSTD_C= ${CC} -c -DZSTD_HEAPMODE=1 -I$S/contrib/zstd/lib/freebsd ${CFLAGS} -I$S/contrib/zstd/lib -I$S/contrib/zstd/lib/common ${WERROR} -Wno-inline -Wno-missing-prototypes ${PROF} -U__BMI__ ${.IMPSRC} +# https://github.com/facebook/zstd/commit/812e8f2a [zstd 1.4.1] +# "Note that [GCC] autovectorization still does not do a good job on the +# optimized version, so it's turned off via attribute and flag. I found +# that neither attribute nor command-line flag were entirely successful in +# turning off vectorization, which is why there were both." +.if ${COMPILER_TYPE} == "gcc" +ZSTD_DECOMPRESS_BLOCK_FLAGS= -fno-tree-vectorize +.endif # Common for dtrace / zfs CDDL_CFLAGS= -DFREEBSD_NAMECACHE -nostdinc -I$S/cddl/compat/opensolaris -I$S/cddl/contrib/opensolaris/uts/common -I$S -I$S/cddl/contrib/opensolaris/common ${CFLAGS} -Wno-unknown-pragmas -Wno-missing-prototypes -Wno-undef -Wno-strict-prototypes -Wno-cast-qual -Wno-parentheses -Wno-redundant-decls -Wno-missing-braces -Wno-uninitialized -Wno-unused -Wno-inline -Wno-switch -Wno-pointer-arith -Wno-unknown-pragmas Modified: head/sys/contrib/zstd/CHANGELOG ============================================================================== --- head/sys/contrib/zstd/CHANGELOG Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/CHANGELOG Thu Aug 8 16:54:22 2019 (r350760) @@ -1,3 +1,39 @@ +v1.4.2 +bug: Fix bug in zstd-0.5 decoder by @terrelln (#1696) +bug: Fix seekable decompression in-memory API by @iburinoc (#1695) +misc: Validate blocks are smaller than size limit by @vivekmg (#1685) +misc: Restructure source files by @ephiepark (#1679) + +v1.4.1 +bug: Fix data corruption in niche use cases by @terrelln (#1659) +bug: Fuzz legacy modes, fix uncovered bugs by @terrelln (#1593, #1594, #1595) +bug: Fix out of bounds read by @terrelln (#1590) +perf: Improve decode speed by ~7% @mgrice (#1668) +perf: Slightly improved compression ratio of level 3 and 4 (ZSTD_dfast) by @cyan4973 (#1681) +perf: Slightly faster compression speed when re-using a context by @cyan4973 (#1658) +perf: Improve compression ratio for small windowLog by @cyan4973 (#1624) +perf: Faster compression speed in high compression mode for repetitive data by @terrelln (#1635) +api: Add parameter to generate smaller dictionaries by @tyler-tran (#1656) +cli: Recognize symlinks when built in C99 mode by @felixhandte (#1640) +cli: Expose cpu load indicator for each file on -vv mode by @ephiepark (#1631) +cli: Restrict read permissions on destination files by @chungy (#1644) +cli: zstdgrep: handle -f flag by @felixhandte (#1618) +cli: zstdcat: follow symlinks by @vejnar (#1604) +doc: Remove extra size limit on compressed blocks by @felixhandte (#1689) +doc: Fix typo by @yk-tanigawa (#1633) +doc: Improve documentation on streaming buffer sizes by @cyan4973 (#1629) +build: CMake: support building with LZ4 @leeyoung624 (#1626) +build: CMake: install zstdless and zstdgrep by @leeyoung624 (#1647) +build: CMake: respect existing uninstall target by @j301scott (#1619) +build: Make: skip multithread tests when built without support by @michaelforney (#1620) +build: Make: Fix examples/ test target by @sjnam (#1603) +build: Meson: rename options out of deprecated namespace by @lzutao (#1665) +build: Meson: fix build by @lzutao (#1602) +build: Visual Studio: don't export symbols in static lib by @scharan (#1650) +build: Visual Studio: fix linking by @absotively (#1639) +build: Fix MinGW-W64 build by @myzhang1029 (#1600) +misc: Expand decodecorpus coverage by @ephiepark (#1664) + v1.4.0 perf: Improve level 1 compression speed in most scenarios by 6% by @gbtucker and @terrelln api: Move the advanced API, including all functions in the staging section, to the stable section Modified: head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c ============================================================================== --- head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c Thu Aug 8 16:54:22 2019 (r350760) @@ -559,7 +559,7 @@ static int benchMem(slice_collection_t dstBlocks, CONTROL(BMK_isSuccessful_runOutcome(outcome)); BMK_runTime_t const result = BMK_extract_runTime(outcome); - U64 const dTime_ns = result.nanoSecPerRun; + double const dTime_ns = result.nanoSecPerRun; double const dTime_sec = (double)dTime_ns / 1000000000; size_t const srcSize = result.sumOfReturn; double const dSpeed_MBps = (double)srcSize / dTime_sec / (1 MB); Modified: head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore ============================================================================== --- head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/contrib/seekable_format/examples/.gitignore Thu Aug 8 16:54:22 2019 (r350760) @@ -1,4 +1,5 @@ seekable_compression seekable_decompression +seekable_decompression_mem parallel_processing parallel_compression Modified: head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile ============================================================================== --- head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile Thu Aug 8 16:54:22 2019 (r350760) @@ -24,7 +24,8 @@ SEEKABLE_OBJS = ../zstdseek_compress.c ../zstdseek_dec default: all -all: seekable_compression seekable_decompression parallel_processing +all: seekable_compression seekable_decompression seekable_decompression_mem \ + parallel_processing $(ZSTDLIB): make -C $(ZSTDLIB_PATH) $(ZSTDLIB_NAME) @@ -35,6 +36,9 @@ seekable_compression : seekable_compression.c $(SEEKAB seekable_decompression : seekable_decompression.c $(SEEKABLE_OBJS) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ +seekable_decompression_mem : seekable_decompression_mem.c $(SEEKABLE_OBJS) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + parallel_processing : parallel_processing.c $(SEEKABLE_OBJS) $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ -pthread @@ -44,5 +48,6 @@ parallel_compression : parallel_compression.c $(SEEKAB clean: @rm -f core *.o tmp* result* *.zst \ seekable_compression seekable_decompression \ + seekable_decompression_mem \ parallel_processing parallel_compression @echo Cleaning completed Copied: head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c (from r350755, vendor/zstd/dist/contrib/seekable_format/examples/seekable_decompression_mem.c) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c Thu Aug 8 16:54:22 2019 (r350760, copy of r350755, vendor/zstd/dist/contrib/seekable_format/examples/seekable_decompression_mem.c) @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2017-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + + +#include <stdlib.h> // malloc, exit +#include <stdio.h> // fprintf, perror, feof +#include <string.h> // strerror +#include <errno.h> // errno +#define ZSTD_STATIC_LINKING_ONLY +#include <zstd.h> // presumes zstd library is installed +#include <zstd_errors.h> + +#include "zstd_seekable.h" + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +#define MAX_FILE_SIZE (8 * 1024 * 1024) + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc"); + exit(1); +} + +static void* realloc_orDie(void* ptr, size_t size) +{ + ptr = realloc(ptr, size); + if (ptr) return ptr; + /* error */ + perror("realloc"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + +static void fseek_orDie(FILE* file, long int offset, int origin) { + if (!fseek(file, offset, origin)) { + if (!fflush(file)) return; + } + /* error */ + perror("fseek"); + exit(7); +} + + +static void decompressFile_orDie(const char* fname, off_t startOffset, off_t endOffset) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = stdout; + // Just for demo purposes, assume file is <= MAX_FILE_SIZE + void* const buffIn = malloc_orDie(MAX_FILE_SIZE); + size_t const inSize = fread_orDie(buffIn, MAX_FILE_SIZE, fin); + size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + void* const buffOut = malloc_orDie(buffOutSize); + + ZSTD_seekable* const seekable = ZSTD_seekable_create(); + if (seekable==NULL) { fprintf(stderr, "ZSTD_seekable_create() error \n"); exit(10); } + + size_t const initResult = ZSTD_seekable_initBuff(seekable, buffIn, inSize); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_seekable_init() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + + while (startOffset < endOffset) { + size_t const result = ZSTD_seekable_decompress(seekable, buffOut, MIN(endOffset - startOffset, buffOutSize), startOffset); + + if (ZSTD_isError(result)) { + fprintf(stderr, "ZSTD_seekable_decompress() error : %s \n", + ZSTD_getErrorName(result)); + exit(12); + } + fwrite_orDie(buffOut, result, fout); + startOffset += result; + } + + ZSTD_seekable_free(seekable); + fclose_orDie(fin); + fclose_orDie(fout); + free(buffIn); + free(buffOut); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=4) { + fprintf(stderr, "wrong arguments\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, "%s FILE START END\n", exeName); + return 1; + } + + { + const char* const inFilename = argv[1]; + off_t const startOffset = atoll(argv[2]); + off_t const endOffset = atoll(argv[3]); + decompressFile_orDie(inFilename, startOffset, endOffset); + } + + return 0; +} Modified: head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c ============================================================================== --- head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c Thu Aug 8 16:54:22 2019 (r350760) @@ -106,7 +106,7 @@ typedef struct { static int ZSTD_seekable_read_buff(void* opaque, void* buffer, size_t n) { buffWrapper_t* buff = (buffWrapper_t*) opaque; - if (buff->size + n > buff->pos) return -1; + if (buff->pos + n > buff->size) return -1; memcpy(buffer, (const BYTE*)buff->ptr + buff->pos, n); buff->pos += n; return 0; @@ -124,7 +124,7 @@ static int ZSTD_seekable_seek_buff(void* opaque, long newOffset = (unsigned long long)buff->pos + offset; break; case SEEK_END: - newOffset = (unsigned long long)buff->size - offset; + newOffset = (unsigned long long)buff->size + offset; break; default: assert(0); /* not possible */ Modified: head/sys/contrib/zstd/doc/zstd_compression_format.md ============================================================================== --- head/sys/contrib/zstd/doc/zstd_compression_format.md Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/doc/zstd_compression_format.md Thu Aug 8 16:54:22 2019 (r350760) @@ -16,7 +16,7 @@ Distribution of this document is unlimited. ### Version -0.3.1 (25/10/18) +0.3.2 (17/07/19) Introduction @@ -390,9 +390,7 @@ A block can contain any number of bytes (even zero), u - Window_Size - 128 KB -A `Compressed_Block` has the extra restriction that `Block_Size` is always -strictly less than the decompressed size. -If this condition cannot be respected, +If this condition cannot be respected when generating a `Compressed_Block`, the block must be sent uncompressed instead (`Raw_Block`). @@ -1655,6 +1653,7 @@ or at least provide a meaningful error code explaining Version changes --------------- +- 0.3.2 : remove additional block size restriction on compressed blocks - 0.3.1 : minor clarification regarding offset history update rules - 0.3.0 : minor edits to match RFC8478 - 0.2.9 : clarifications for huffman weights direct representation, by Ulrich Kunitz Modified: head/sys/contrib/zstd/doc/zstd_manual.html ============================================================================== --- head/sys/contrib/zstd/doc/zstd_manual.html Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/doc/zstd_manual.html Thu Aug 8 16:54:22 2019 (r350760) @@ -1,46 +1,36 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>zstd 1.4.0 Manual</title> +<title>zstd 1.4.2 Manual</title> </head> <body> -<h1>zstd 1.4.0 Manual</h1> +<h1>zstd 1.4.2 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> <li><a href="#Chapter1">Introduction</a></li> <li><a href="#Chapter2">Version</a></li> -<li><a href="#Chapter3">Default constant</a></li> -<li><a href="#Chapter4">Constants</a></li> -<li><a href="#Chapter5">Simple API</a></li> -<li><a href="#Chapter6">Explicit context</a></li> -<li><a href="#Chapter7">Advanced compression API</a></li> -<li><a href="#Chapter8">Advanced decompression API</a></li> -<li><a href="#Chapter9">Streaming</a></li> -<li><a href="#Chapter10">Streaming compression - HowTo</a></li> -<li><a href="#Chapter11">This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</a></li> -<li><a href="#Chapter12">Equivalent to:</a></li> -<li><a href="#Chapter13">Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</a></li> -<li><a href="#Chapter14">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</a></li> -<li><a href="#Chapter15">Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</a></li> -<li><a href="#Chapter16">Streaming decompression - HowTo</a></li> -<li><a href="#Chapter17">Simple dictionary API</a></li> -<li><a href="#Chapter18">Bulk processing dictionary API</a></li> -<li><a href="#Chapter19">Dictionary helper functions</a></li> -<li><a href="#Chapter20">Advanced dictionary and prefix API</a></li> -<li><a href="#Chapter21">ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> -<li><a href="#Chapter22">experimental API (static linking only)</a></li> -<li><a href="#Chapter23">Frame size functions</a></li> -<li><a href="#Chapter24">ZSTD_decompressBound() :</a></li> -<li><a href="#Chapter25">Memory management</a></li> -<li><a href="#Chapter26">Advanced compression functions</a></li> -<li><a href="#Chapter27">Advanced decompression functions</a></li> -<li><a href="#Chapter28">Advanced streaming functions</a></li> -<li><a href="#Chapter29">Buffer-less and synchronous inner streaming functions</a></li> -<li><a href="#Chapter30">Buffer-less streaming compression (synchronous mode)</a></li> -<li><a href="#Chapter31">Buffer-less streaming decompression (synchronous mode)</a></li> -<li><a href="#Chapter32">ZSTD_getFrameHeader() :</a></li> -<li><a href="#Chapter33">Block level API</a></li> +<li><a href="#Chapter3">Simple API</a></li> +<li><a href="#Chapter4">Explicit context</a></li> +<li><a href="#Chapter5">Advanced compression API</a></li> +<li><a href="#Chapter6">Advanced decompression API</a></li> +<li><a href="#Chapter7">Streaming</a></li> +<li><a href="#Chapter8">Streaming compression - HowTo</a></li> +<li><a href="#Chapter9">Streaming decompression - HowTo</a></li> +<li><a href="#Chapter10">Simple dictionary API</a></li> +<li><a href="#Chapter11">Bulk processing dictionary API</a></li> +<li><a href="#Chapter12">Dictionary helper functions</a></li> +<li><a href="#Chapter13">Advanced dictionary and prefix API</a></li> +<li><a href="#Chapter14">experimental API (static linking only)</a></li> +<li><a href="#Chapter15">Frame size functions</a></li> +<li><a href="#Chapter16">Memory management</a></li> +<li><a href="#Chapter17">Advanced compression functions</a></li> +<li><a href="#Chapter18">Advanced decompression functions</a></li> +<li><a href="#Chapter19">Advanced streaming functions</a></li> +<li><a href="#Chapter20">Buffer-less and synchronous inner streaming functions</a></li> +<li><a href="#Chapter21">Buffer-less streaming compression (synchronous mode)</a></li> +<li><a href="#Chapter22">Buffer-less streaming decompression (synchronous mode)</a></li> +<li><a href="#Chapter23">Block level API</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -78,12 +68,8 @@ <pre><b>unsigned ZSTD_versionNumber(void); </b>/**< to check runtime library version */<b> </b></pre><BR> -<a name="Chapter3"></a><h2>Default constant</h2><pre></pre> +<a name="Chapter3"></a><h2>Simple API</h2><pre></pre> -<a name="Chapter4"></a><h2>Constants</h2><pre></pre> - -<a name="Chapter5"></a><h2>Simple API</h2><pre></pre> - <pre><b>size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); @@ -152,12 +138,17 @@ const char* ZSTD_getErrorName(size_t code); </b>/* int ZSTD_minCLevel(void); </b>/*!< minimum negative compression level allowed */<b> int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b> </pre></b><BR> -<a name="Chapter6"></a><h2>Explicit context</h2><pre></pre> +<a name="Chapter4"></a><h2>Explicit context</h2><pre></pre> <h3>Compression context</h3><pre> When compressing many times, - it is recommended to allocate a context just once, and re-use it for each successive compression operation. + it is recommended to allocate a context just once, + and re-use it for each successive compression operation. This will make workload friendlier for system's memory. - Use one context per thread for parallel execution in multi-threaded environments. + Note : re-using context is just a speed / resource optimization. + It doesn't change the compression ratio, which remains identical. + Note 2 : In multi-threaded environments, + use one different context per thread for parallel execution. + </pre><b><pre>typedef struct ZSTD_CCtx_s ZSTD_CCtx; ZSTD_CCtx* ZSTD_createCCtx(void); size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); @@ -189,7 +180,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </p></pre><BR> -<a name="Chapter7"></a><h2>Advanced compression API</h2><pre></pre> +<a name="Chapter5"></a><h2>Advanced compression API</h2><pre></pre> <pre><b>typedef enum { ZSTD_fast=1, ZSTD_dfast=2, @@ -332,6 +323,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); * ZSTD_c_forceMaxWindow * ZSTD_c_forceAttachDict * ZSTD_c_literalCompressionMode + * ZSTD_c_targetCBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -341,6 +333,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); ZSTD_c_experimentalParam3=1000, ZSTD_c_experimentalParam4=1001, ZSTD_c_experimentalParam5=1002, + ZSTD_c_experimentalParam6=1003, } ZSTD_cParameter; </b></pre><BR> <pre><b>typedef struct { @@ -424,7 +417,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </p></pre><BR> -<a name="Chapter8"></a><h2>Advanced decompression API</h2><pre></pre> +<a name="Chapter6"></a><h2>Advanced decompression API</h2><pre></pre> <pre><b>typedef enum { @@ -472,7 +465,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); </p></pre><BR> -<a name="Chapter9"></a><h2>Streaming</h2><pre></pre> +<a name="Chapter7"></a><h2>Streaming</h2><pre></pre> <pre><b>typedef struct ZSTD_inBuffer_s { const void* src; </b>/**< start of input buffer */<b> @@ -486,7 +479,7 @@ size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); size_t pos; </b>/**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */<b> } ZSTD_outBuffer; </b></pre><BR> -<a name="Chapter10"></a><h2>Streaming compression - HowTo</h2><pre> +<a name="Chapter8"></a><h2>Streaming compression - HowTo</h2><pre> A ZSTD_CStream object is required to track streaming operation. Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. ZSTD_CStream objects can be reused multiple times on consecutive compression operations. @@ -592,31 +585,28 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs); <pre><b>size_t ZSTD_CStreamInSize(void); </b>/**< recommended size for input buffer */<b> </b></pre><BR> -<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */<b> +<pre><b>size_t ZSTD_CStreamOutSize(void); </b>/**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */<b> </b></pre><BR> -<a name="Chapter11"></a><h2>This is a legacy streaming API, and can be replaced by ZSTD_CCtx_reset() and</h2><pre> ZSTD_compressStream2(). It is redundent, but is still fully supported. - Advanced parameters and dictionary compression can only be used through the - new API. -<BR></pre> - -<a name="Chapter12"></a><h2>Equivalent to:</h2><pre> +<pre><b>size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +</b>/*!<b> + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +</b>/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */<b> +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +</b>/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */<b> +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +</b><p> ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); -<BR></pre> +</p></pre><BR> -<a name="Chapter13"></a><h2>Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue).</h2><pre> NOTE: The return value is different. ZSTD_compressStream() returns a hint for - the next read size (if non-zero and not an error). ZSTD_compressStream2() - returns the number of bytes left to flush (if non-zero and not an error). - -<BR></pre> - -<a name="Chapter14"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush).</h2><pre></pre> - -<a name="Chapter15"></a><h2>Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end).</h2><pre></pre> - -<a name="Chapter16"></a><h2>Streaming decompression - HowTo</h2><pre> +<a name="Chapter9"></a><h2>Streaming decompression - HowTo</h2><pre> A ZSTD_DStream object is required to track streaming operations. Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. ZSTD_DStream objects can be re-used multiple times. @@ -647,14 +637,12 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs); <h3>ZSTD_DStream management functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream(void); size_t ZSTD_freeDStream(ZSTD_DStream* zds); </pre></b><BR> -<h3>Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream(ZSTD_DStream* zds); -size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); -</pre></b><BR> +<h3>Streaming decompression functions</h3><pre></pre><b><pre></pre></b><BR> <pre><b>size_t ZSTD_DStreamInSize(void); </b>/*!< recommended size for input buffer */<b> </b></pre><BR> <pre><b>size_t ZSTD_DStreamOutSize(void); </b>/*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */<b> </b></pre><BR> -<a name="Chapter17"></a><h2>Simple dictionary API</h2><pre></pre> +<a name="Chapter10"></a><h2>Simple dictionary API</h2><pre></pre> <pre><b>size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, @@ -680,7 +668,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o Note : When `dict == NULL || dictSize < 8` no dictionary is used. </p></pre><BR> -<a name="Chapter18"></a><h2>Bulk processing dictionary API</h2><pre></pre> +<a name="Chapter11"></a><h2>Bulk processing dictionary API</h2><pre></pre> <pre><b>ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); @@ -723,7 +711,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o Recommended when same dictionary is used multiple times. </p></pre><BR> -<a name="Chapter19"></a><h2>Dictionary helper functions</h2><pre></pre> +<a name="Chapter12"></a><h2>Dictionary helper functions</h2><pre></pre> <pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); </b><p> Provides the dictID stored within dictionary. @@ -749,7 +737,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. </p></pre><BR> -<a name="Chapter20"></a><h2>Advanced dictionary and prefix API</h2><pre> +<a name="Chapter13"></a><h2>Advanced dictionary and prefix API</h2><pre> This API allows dictionaries to be used with ZSTD_compress2(), ZSTD_compressStream2(), and ZSTD_decompress(). Dictionaries are sticky, and only reset with the context is reset with ZSTD_reset_parameters or @@ -867,15 +855,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); Note that object memory usage can evolve (increase or decrease) over time. </p></pre><BR> -<a name="Chapter21"></a><h2>ADVANCED AND EXPERIMENTAL FUNCTIONS</h2><pre> - The definitions in the following section are considered experimental. - They are provided for advanced scenarios. - They should never be used with a dynamic library, as prototypes may change in the future. - Use them only in association with static linking. - -<BR></pre> - -<a name="Chapter22"></a><h2>experimental API (static linking only)</h2><pre> +<a name="Chapter14"></a><h2>experimental API (static linking only)</h2><pre> The following symbols and constants are not planned to join "stable API" status in the near future. They can still change in future versions. @@ -973,7 +953,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTD_lcm_uncompressed = 2, </b>/**< Always emit uncompressed literals. */<b> } ZSTD_literalCompressionMode_e; </b></pre><BR> -<a name="Chapter23"></a><h2>Frame size functions</h2><pre></pre> +<a name="Chapter15"></a><h2>Frame size functions</h2><pre></pre> <pre><b>unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); </b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames @@ -998,7 +978,8 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); however it does mean that all frame data must be present and valid. </p></pre><BR> -<a name="Chapter24"></a><h2>ZSTD_decompressBound() :</h2><pre> `src` should point to the start of a series of ZSTD encoded and/or skippable frames +<pre><b>unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); +</b><p> `src` should point to the start of a series of ZSTD encoded and/or skippable frames `srcSize` must be the _exact_ size of this series (i.e. there should be a frame boundary at `src + srcSize`) @return : - upper-bound for the decompressed size of all data in all successive frames @@ -1010,7 +991,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: upper-bound = # blocks * min(128 KB, Window_Size) -<BR></pre> +</p></pre><BR> <pre><b>size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); </b><p> srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. @@ -1018,7 +999,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); or an error code (if srcSize is too small) </p></pre><BR> -<a name="Chapter25"></a><h2>Memory management</h2><pre></pre> +<a name="Chapter16"></a><h2>Memory management</h2><pre></pre> <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); @@ -1098,7 +1079,7 @@ static ZSTD_customMem const ZSTD_defaultCMem = { NULL, </p></pre><BR> -<a name="Chapter26"></a><h2>Advanced compression functions</h2><pre></pre> +<a name="Chapter17"></a><h2>Advanced compression functions</h2><pre></pre> <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); </b><p> Create a digested dictionary for compression @@ -1243,7 +1224,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> -<a name="Chapter27"></a><h2>Advanced decompression functions</h2><pre></pre> +<a name="Chapter18"></a><h2>Advanced decompression functions</h2><pre></pre> <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. @@ -1305,7 +1286,7 @@ size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); </p></pre><BR> -<a name="Chapter28"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API. +<a name="Chapter19"></a><h2>Advanced streaming functions</h2><pre> Warning : most of these functions are now redundant with the Advanced API. Once Advanced API reaches "stable" status, redundant functions will be deprecated, and then at some point removed. <BR></pre> @@ -1407,18 +1388,41 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStre </p></pre><BR> -<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b> -size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b> -size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b> +<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre></b>/**<b> + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +</b>/**<b> + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); +</b>/**<b> + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * re-use decompression parameters from previous init; saves dictionary loading + */ +size_t ZSTD_resetDStream(ZSTD_DStream* zds); </pre></b><BR> -<a name="Chapter29"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> +<a name="Chapter20"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> This is an advanced API, giving full control over buffer management, for users which need direct control over memory. But it's also a complex one, with several restrictions, documented below. Prefer normal streaming API for an easier experience. <BR></pre> -<a name="Chapter30"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> +<a name="Chapter21"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. ZSTD_CCtx object can be re-used multiple times within successive compression operations. @@ -1454,7 +1458,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> </pre></b><BR> -<a name="Chapter31"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> +<a name="Chapter22"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. @@ -1536,23 +1540,21 @@ typedef struct { unsigned checksumFlag; } ZSTD_frameHeader; </pre></b><BR> -<a name="Chapter32"></a><h2>ZSTD_getFrameHeader() :</h2><pre> decode Frame Header, or requires larger `srcSize`. +<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b> +</b>/*! ZSTD_getFrameHeader_advanced() :<b> + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b> +</b><p> decode Frame Header, or requires larger `srcSize`. @return : 0, `zfhPtr` is correctly filled, >0, `srcSize` is too small, value is wanted `srcSize` amount, or an error code, which can be tested using ZSTD_isError() -<BR></pre> - -<pre><b>size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); </b>/**< doesn't consume input */<b> -</b></pre><BR> -<pre><b>size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); -size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); </b>/**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */<b> -</b><p> same as ZSTD_getFrameHeader(), - with added capability to select a format (like ZSTD_f_zstd1_magicless) </p></pre><BR> <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; </b></pre><BR> -<a name="Chapter33"></a><h2>Block level API</h2><pre></pre> +<a name="Chapter23"></a><h2>Block level API</h2><pre></pre> <pre><b></b><p> Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). User will have to take in charge required information to regenerate data, such as compressed and content sizes. Modified: head/sys/contrib/zstd/examples/Makefile ============================================================================== --- head/sys/contrib/zstd/examples/Makefile Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/examples/Makefile Thu Aug 8 16:54:22 2019 (r350760) @@ -77,7 +77,6 @@ test: all @echo -- Edge cases detection ! ./streaming_decompression tmp # invalid input, must fail ! ./simple_decompression tmp # invalid input, must fail - ! ./simple_decompression tmp.zst # unknown input size, must fail touch tmpNull # create 0-size file ./simple_compression tmpNull ./simple_decompression tmpNull.zst # 0-size frame : must work Modified: head/sys/contrib/zstd/lib/Makefile ============================================================================== --- head/sys/contrib/zstd/lib/Makefile Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/lib/Makefile Thu Aug 8 16:54:22 2019 (r350760) @@ -17,6 +17,7 @@ LIBVER_MINOR := $(shell echo $(LIBVER_MINOR_SCRIPT)) LIBVER_PATCH := $(shell echo $(LIBVER_PATCH_SCRIPT)) LIBVER := $(shell echo $(LIBVER_SCRIPT)) VERSION?= $(LIBVER) +CCVER := $(shell $(CC) --version) CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_ ifeq ($(OS),Windows_NT) # MinGW assumed @@ -44,6 +45,10 @@ ZSTDDECOMP_FILES := $(sort $(wildcard decompress/*.c)) ZDICT_FILES := $(sort $(wildcard dictBuilder/*.c)) ZDEPR_FILES := $(sort $(wildcard deprecated/*.c)) ZSTD_FILES := $(ZSTDCOMMON_FILES) + +ifeq ($(findstring GCC,$(CCVER)),GCC) +decompress/zstd_decompress_block.o : CFLAGS+=-fno-tree-vectorize +endif ZSTD_LEGACY_SUPPORT ?= 5 ZSTD_LIB_COMPRESSION ?= 1 Modified: head/sys/contrib/zstd/lib/common/compiler.h ============================================================================== --- head/sys/contrib/zstd/lib/common/compiler.h Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/lib/common/compiler.h Thu Aug 8 16:54:22 2019 (r350760) @@ -127,6 +127,13 @@ } \ } +/* vectorization */ +#if !defined(__clang__) && defined(__GNUC__) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +#else +# define DONT_VECTORIZE +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include <intrin.h> /* For Visual 2005 */ Modified: head/sys/contrib/zstd/lib/common/zstd_internal.h ============================================================================== --- head/sys/contrib/zstd/lib/common/zstd_internal.h Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/lib/common/zstd_internal.h Thu Aug 8 16:54:22 2019 (r350760) @@ -34,7 +34,6 @@ #endif #include "xxhash.h" /* XXH_reset, update, digest */ - #if defined (__cplusplus) extern "C" { #endif @@ -193,19 +192,72 @@ static const U32 OF_defaultNormLog = OF_DEFAULTNORMLOG * Shared functions to include for inlining *********************************************/ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); } + #define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; } +static void ZSTD_copy16(void* dst, const void* src) { memcpy(dst, src, 16); } +#define COPY16(d,s) { ZSTD_copy16(d,s); d+=16; s+=16; } +#define WILDCOPY_OVERLENGTH 8 +#define VECLEN 16 + +typedef enum { + ZSTD_no_overlap, + ZSTD_overlap_src_before_dst, + /* ZSTD_overlap_dst_before_src, */ +} ZSTD_overlap_e; + /*! ZSTD_wildcopy() : * custom version of memcpy(), can overwrite up to WILDCOPY_OVERLENGTH bytes (if length==0) */ -#define WILDCOPY_OVERLENGTH 8 -MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length) +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; BYTE* const oend = op + length; - do - COPY8(op, ip) - while (op < oend); + + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + if (length < VECLEN || (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN)) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } +} + +/*! ZSTD_wildcopy_16min() : + * same semantics as ZSTD_wilcopy() except guaranteed to be able to copy 16 bytes at the start */ +MEM_STATIC FORCE_INLINE_ATTR DONT_VECTORIZE +void ZSTD_wildcopy_16min(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e ovtype) +{ + ptrdiff_t diff = (BYTE*)dst - (const BYTE*)src; + const BYTE* ip = (const BYTE*)src; + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + length; + + assert(length >= 8); + assert(diff >= 8 || (ovtype == ZSTD_no_overlap && diff < -8)); + + if (ovtype == ZSTD_overlap_src_before_dst && diff < VECLEN) { + do + COPY8(op, ip) + while (op < oend); + } + else { + if ((length & 8) == 0) + COPY8(op, ip); + do { + COPY16(op, ip); + } + while (op < oend); + } } MEM_STATIC void ZSTD_wildcopy_e(void* dst, const void* src, void* dstEnd) /* should be faster for decoding, but strangely, not verified on all platform */ Modified: head/sys/contrib/zstd/lib/compress/zstd_compress.c ============================================================================== --- head/sys/contrib/zstd/lib/compress/zstd_compress.c Thu Aug 8 16:51:49 2019 (r350759) +++ head/sys/contrib/zstd/lib/compress/zstd_compress.c Thu Aug 8 16:54:22 2019 (r350760) @@ -21,6 +21,8 @@ #define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_compress_internal.h" +#include "zstd_compress_sequences.h" +#include "zstd_compress_literals.h" #include "zstd_fast.h" #include "zstd_double_fast.h" #include "zstd_lazy.h" @@ -385,6 +387,11 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter para bounds.upperBound = ZSTD_lcm_uncompressed; return bounds; + case ZSTD_c_targetCBlockSize: + bounds.lowerBound = ZSTD_TARGETCBLOCKSIZE_MIN; + bounds.upperBound = ZSTD_TARGETCBLOCKSIZE_MAX; + return bounds; + default: { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; return boundError; @@ -392,18 +399,6 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter para } } -/* ZSTD_cParam_withinBounds: - * @return 1 if value is within cParam bounds, - * 0 otherwise */ -static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) -{ - ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); - if (ZSTD_isError(bounds.error)) return 0; - if (value < bounds.lowerBound) return 0; - if (value > bounds.upperBound) return 0; - return 1; -} - /* ZSTD_cParam_clampBounds: * Clamps the value into the bounded range. */ @@ -452,6 +447,7 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter par case ZSTD_c_ldmHashRateLog: case ZSTD_c_forceAttachDict: case ZSTD_c_literalCompressionMode: + case ZSTD_c_targetCBlockSize: default: return 0; } @@ -497,6 +493,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cP case ZSTD_c_ldmHashLog: case ZSTD_c_ldmMinMatch: case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_targetCBlockSize: break; default: RETURN_ERROR(parameter_unsupported); @@ -671,6 +668,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams->ldmParams.hashRateLog = value; return CCtxParams->ldmParams.hashRateLog; + case ZSTD_c_targetCBlockSize : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_targetCBlockSize, value); + CCtxParams->targetCBlockSize = value; + return CCtxParams->targetCBlockSize; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -692,13 +695,13 @@ size_t ZSTD_CCtxParams_getParameter( *value = CCtxParams->compressionLevel; break; case ZSTD_c_windowLog : - *value = CCtxParams->cParams.windowLog; + *value = (int)CCtxParams->cParams.windowLog; break; case ZSTD_c_hashLog : - *value = CCtxParams->cParams.hashLog; + *value = (int)CCtxParams->cParams.hashLog; break; case ZSTD_c_chainLog : - *value = CCtxParams->cParams.chainLog; + *value = (int)CCtxParams->cParams.chainLog; break; case ZSTD_c_searchLog : *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201908081654.x78GsMU6055994>