Date: Mon, 22 Oct 2018 19:50:43 +0000 (UTC) From: Conrad Meyer <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-vendor@freebsd.org Subject: svn commit: r339612 - in vendor/zstd/dist: . contrib/adaptive-compression contrib/gen_html contrib/long_distance_matching contrib/meson contrib/seekable_format doc doc/images lib lib/common lib/com... Message-ID: <201810221950.w9MJohXr076158@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Mon Oct 22 19:50:43 2018 New Revision: 339612 URL: https://svnweb.freebsd.org/changeset/base/339612 Log: import zstd 1.3.4 Added: vendor/zstd/dist/doc/images/CSpeed2.png (contents, props changed) vendor/zstd/dist/doc/images/DSpeed3.png (contents, props changed) vendor/zstd/dist/doc/images/linux-4.7-12-mt-compress.png (contents, props changed) vendor/zstd/dist/doc/images/linux-git-mt-compress.png (contents, props changed) vendor/zstd/dist/lib/common/cpu.h (contents, props changed) vendor/zstd/dist/tests/checkTag.c (contents, props changed) Deleted: vendor/zstd/dist/contrib/long_distance_matching/ vendor/zstd/dist/doc/images/Cspeed4.png vendor/zstd/dist/doc/images/Dspeed4.png vendor/zstd/dist/tests/namespaceTest.c Modified: vendor/zstd/dist/Makefile vendor/zstd/dist/NEWS vendor/zstd/dist/README.md vendor/zstd/dist/appveyor.yml vendor/zstd/dist/contrib/adaptive-compression/adapt.c vendor/zstd/dist/contrib/gen_html/Makefile vendor/zstd/dist/contrib/meson/meson.build vendor/zstd/dist/contrib/meson/meson_options.txt vendor/zstd/dist/contrib/seekable_format/zstdseek_compress.c vendor/zstd/dist/contrib/seekable_format/zstdseek_decompress.c vendor/zstd/dist/doc/README.md vendor/zstd/dist/doc/images/dict-cr.png vendor/zstd/dist/doc/images/dict-cs.png vendor/zstd/dist/doc/images/dict-ds.png vendor/zstd/dist/doc/zstd_compression_format.md vendor/zstd/dist/doc/zstd_manual.html vendor/zstd/dist/lib/BUCK vendor/zstd/dist/lib/README.md vendor/zstd/dist/lib/common/bitstream.h vendor/zstd/dist/lib/common/compiler.h vendor/zstd/dist/lib/common/error_private.c vendor/zstd/dist/lib/common/fse.h vendor/zstd/dist/lib/common/fse_decompress.c vendor/zstd/dist/lib/common/huf.h vendor/zstd/dist/lib/common/pool.c vendor/zstd/dist/lib/common/pool.h vendor/zstd/dist/lib/common/threading.h vendor/zstd/dist/lib/common/zstd_errors.h vendor/zstd/dist/lib/common/zstd_internal.h vendor/zstd/dist/lib/compress/fse_compress.c vendor/zstd/dist/lib/compress/huf_compress.c vendor/zstd/dist/lib/compress/zstd_compress.c vendor/zstd/dist/lib/compress/zstd_compress_internal.h vendor/zstd/dist/lib/compress/zstd_double_fast.c vendor/zstd/dist/lib/compress/zstd_double_fast.h vendor/zstd/dist/lib/compress/zstd_fast.c vendor/zstd/dist/lib/compress/zstd_fast.h vendor/zstd/dist/lib/compress/zstd_lazy.c vendor/zstd/dist/lib/compress/zstd_lazy.h vendor/zstd/dist/lib/compress/zstd_ldm.c vendor/zstd/dist/lib/compress/zstd_ldm.h vendor/zstd/dist/lib/compress/zstd_opt.c vendor/zstd/dist/lib/compress/zstd_opt.h vendor/zstd/dist/lib/compress/zstdmt_compress.c vendor/zstd/dist/lib/compress/zstdmt_compress.h vendor/zstd/dist/lib/decompress/huf_decompress.c vendor/zstd/dist/lib/decompress/zstd_decompress.c vendor/zstd/dist/lib/dictBuilder/cover.c vendor/zstd/dist/lib/dictBuilder/zdict.c vendor/zstd/dist/lib/dictBuilder/zdict.h vendor/zstd/dist/lib/legacy/zstd_legacy.h vendor/zstd/dist/lib/legacy/zstd_v04.c vendor/zstd/dist/lib/legacy/zstd_v06.c vendor/zstd/dist/lib/legacy/zstd_v07.c vendor/zstd/dist/lib/zstd.h vendor/zstd/dist/programs/Makefile vendor/zstd/dist/programs/README.md vendor/zstd/dist/programs/bench.c vendor/zstd/dist/programs/bench.h vendor/zstd/dist/programs/fileio.c vendor/zstd/dist/programs/fileio.h vendor/zstd/dist/programs/platform.h vendor/zstd/dist/programs/util.h vendor/zstd/dist/programs/zstd.1 vendor/zstd/dist/programs/zstd.1.md vendor/zstd/dist/programs/zstdcli.c vendor/zstd/dist/tests/.gitignore vendor/zstd/dist/tests/Makefile vendor/zstd/dist/tests/fullbench.c vendor/zstd/dist/tests/fuzz/zstd_helpers.c vendor/zstd/dist/tests/fuzzer.c vendor/zstd/dist/tests/legacy.c vendor/zstd/dist/tests/paramgrill.c vendor/zstd/dist/tests/playTests.sh vendor/zstd/dist/tests/roundTripCrash.c vendor/zstd/dist/tests/zstreamtest.c vendor/zstd/dist/zlibWrapper/examples/zwrapbench.c Modified: vendor/zstd/dist/Makefile ============================================================================== --- vendor/zstd/dist/Makefile Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/Makefile Mon Oct 22 19:50:43 2018 (r339612) @@ -27,7 +27,7 @@ endif default: lib-release zstd-release .PHONY: all -all: | allmost examples manual +all: | allmost examples manual contrib .PHONY: allmost allmost: allzstd @@ -72,14 +72,18 @@ zstdmt: zlibwrapper: $(MAKE) -C $(ZWRAPDIR) test -.PHONY: check -check: shortest +.PHONY: test +test: + $(MAKE) -C $(PRGDIR) allVariants MOREFLAGS+="-g -DZSTD_DEBUG=1" + $(MAKE) -C $(TESTDIR) $@ -.PHONY: test shortest -test shortest: - $(MAKE) -C $(PRGDIR) allVariants MOREFLAGS="-g -DZSTD_DEBUG=1" +.PHONY: shortest +shortest: $(MAKE) -C $(TESTDIR) $@ +.PHONY: check +check: shortest + .PHONY: examples examples: CPPFLAGS=-I../lib LDFLAGS=-L../lib $(MAKE) -C examples/ all @@ -88,6 +92,12 @@ examples: manual: $(MAKE) -C contrib/gen_html $@ +.PHONY: contrib +contrib: lib + $(MAKE) -C contrib/pzstd all + $(MAKE) -C contrib/seekable_format/examples all + $(MAKE) -C contrib/adaptive-compression all + .PHONY: cleanTabs cleanTabs: cd contrib; ./cleanTabs @@ -100,6 +110,9 @@ clean: @$(MAKE) -C $(ZWRAPDIR) $@ > $(VOID) @$(MAKE) -C examples/ $@ > $(VOID) @$(MAKE) -C contrib/gen_html $@ > $(VOID) + @$(MAKE) -C contrib/pzstd $@ > $(VOID) + @$(MAKE) -C contrib/seekable_format/examples $@ > $(VOID) + @$(MAKE) -C contrib/adaptive-compression $@ > $(VOID) @$(RM) zstd$(EXT) zstdmt$(EXT) tmp* @$(RM) -r lz4 @echo Cleaning completed @@ -231,31 +244,31 @@ msanregressiontest: # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63303 usan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined" + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=undefined -Werror" asan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address" + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=address -Werror" asan-%: clean - LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address" $(MAKE) -C $(TESTDIR) $* + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=address -Werror" $(MAKE) -C $(TESTDIR) $* msan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason + $(MAKE) test CC=clang MOREFLAGS="-g -fsanitize=memory -fno-omit-frame-pointer -Werror" HAVE_LZMA=0 # datagen.c fails this test for no obvious reason msan-%: clean - LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $* + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=memory -fno-omit-frame-pointer -Werror" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) HAVE_LZMA=0 $* asan32: clean $(MAKE) -C $(TESTDIR) test32 CC=clang MOREFLAGS="-g -fsanitize=address" uasan: clean - $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" + $(MAKE) test CC=clang MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" uasan-%: clean - LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined" $(MAKE) -C $(TESTDIR) $* + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize-recover=signed-integer-overflow -fsanitize=address,undefined -Werror" $(MAKE) -C $(TESTDIR) $* tsan-%: clean - LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests + LDFLAGS=-fuse-ld=gold MOREFLAGS="-g -fno-sanitize-recover=all -fsanitize=thread -Werror" $(MAKE) -C $(TESTDIR) $* FUZZER_FLAGS=--no-big-tests apt-install: sudo apt-get -yq --no-install-suggests --no-install-recommends --force-yes install $(APT_PACKAGES) @@ -278,6 +291,9 @@ libc6install: gcc6install: apt-add-repo APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-6 gcc-6-multilib" $(MAKE) apt-install + +gcc7install: apt-add-repo + APT_PACKAGES="libc6-dev-i386 gcc-multilib gcc-7 gcc-7-multilib" $(MAKE) apt-install gpp6install: apt-add-repo APT_PACKAGES="libc6-dev-i386 g++-multilib gcc-6 g++-6 g++-6-multilib" $(MAKE) apt-install Modified: vendor/zstd/dist/NEWS ============================================================================== --- vendor/zstd/dist/NEWS Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/NEWS Mon Oct 22 19:50:43 2018 (r339612) @@ -1,5 +1,23 @@ +v1.3.4 +perf: faster speed (especially decoding speed) on recent cpus (haswell+) +perf: much better performance associating --long with multi-threading, by @terrelln +perf: better compression at levels 13-15 +cli : asynchronous compression by default, for faster experience (use --single-thread for former behavior) +cli : smoother status report in multi-threading mode +cli : added command --fast=#, for faster compression modes +cli : fix crash when not overwriting existing files, by Pádraig Brady (@pixelb) +api : `nbThreads` becomes `nbWorkers` : 1 triggers asynchronous mode +api : compression levels can be negative, for even more speed +api : ZSTD_getFrameProgression() : get precise progress status of ZSTDMT anytime +api : ZSTDMT can accept new compression parameters during compression +api : implemented all advanced dictionary decompression prototypes +build: improved meson recipe, by Shawn Landden (@shawnl) +build: VS2017 scripts, by @HaydnTrigg +misc: all /contrib projects fixed +misc: added /contrib/docker script by @gyscos + v1.3.3 -perf: faster zstd_opt strategy (levels 17-19) +perf: faster zstd_opt strategy (levels 16-19) fix : bug #944 : multithreading with shared ditionary and large data, reported by @gsliepen cli : fix : content size written in header by default cli : fix : improved LZ4 format support, by @felixhandte Modified: vendor/zstd/dist/README.md ============================================================================== --- vendor/zstd/dist/README.md Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/README.md Mon Oct 22 19:50:43 2018 (r339612) @@ -1,4 +1,4 @@ -<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/readme/doc/images/zstd_logo86.png" alt="Zstandard"></p> +<p align="center"><img src="https://raw.githubusercontent.com/facebook/zstd/dev/doc/images/zstd_logo86.png" alt="Zstandard"></p> __Zstandard__, or `zstd` as short version, is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level and better compression ratios. @@ -21,24 +21,25 @@ Development branch status : [![Build Status][travisDev ### Benchmarks For reference, several fast compression algorithms were tested and compared -on a server running Linux Debian (`Linux version 4.8.0-1-amd64`), +on a server running Linux Debian (`Linux version 4.14.0-3-amd64`), with a Core i7-6700K CPU @ 4.0GHz, using [lzbench], an open-source in-memory benchmark by @inikep -compiled with GCC 6.3.0, +compiled with [gcc] 7.3.0, on the [Silesia compression corpus]. [lzbench]: https://github.com/inikep/lzbench [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia +[gcc]: https://gcc.gnu.org/ | Compressor name | Ratio | Compression| Decompress.| | --------------- | ------| -----------| ---------- | -| **zstd 1.1.3 -1** | 2.877 | 430 MB/s | 1110 MB/s | -| zlib 1.2.8 -1 | 2.743 | 110 MB/s | 400 MB/s | -| brotli 0.5.2 -0 | 2.708 | 400 MB/s | 430 MB/s | +| **zstd 1.3.4 -1** | 2.877 | 470 MB/s | 1380 MB/s | +| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 400 MB/s | +| brotli 1.0.2 -0 | 2.701 | 410 MB/s | 430 MB/s | | quicklz 1.5.0 -1 | 2.238 | 550 MB/s | 710 MB/s | | lzo1x 2.09 -1 | 2.108 | 650 MB/s | 830 MB/s | -| lz4 1.7.5 | 2.101 | 720 MB/s | 3600 MB/s | -| snappy 1.1.3 | 2.091 | 500 MB/s | 1650 MB/s | +| lz4 1.8.1 | 2.101 | 750 MB/s | 3700 MB/s | +| snappy 1.1.4 | 2.091 | 530 MB/s | 1800 MB/s | | lzf 3.6 -1 | 2.077 | 400 MB/s | 860 MB/s | [zlib]:http://www.zlib.net/ @@ -50,15 +51,15 @@ Decompression speed is preserved and remains roughly t a property shared by most LZ compression algorithms, such as [zlib] or lzma. The following tests were run -on a server running Linux Debian (`Linux version 4.8.0-1-amd64`) +on a server running Linux Debian (`Linux version 4.14.0-3-amd64`) with a Core i7-6700K CPU @ 4.0GHz, using [lzbench], an open-source in-memory benchmark by @inikep -compiled with GCC 6.3.0, +compiled with [gcc] 7.3.0, on the [Silesia compression corpus]. Compression Speed vs Ratio | Decompression Speed ---------------------------|-------------------- - |  + |  A few other algorithms can produce higher compression ratios at slower speeds, falling outside of the graph. For a larger picture including slow modes, [click on this link](doc/images/DCspeed5.png). @@ -128,8 +129,8 @@ A Meson project is provided within `contrib/meson`. Going into `build` directory, you will find additional possibilities: - Projects for Visual Studio 2005, 2008 and 2010. - + VS2010 project is compatible with VS2012, VS2013 and VS2015. -- Automated build scripts for Visual compiler by @KrzysFR , in `build/VS_scripts`, + + VS2010 project is compatible with VS2012, VS2013, VS2015 and VS2017. +- Automated build scripts for Visual compiler by [@KrzysFR](https://github.com/KrzysFR), in `build/VS_scripts`, which will build `zstd` cli and `libzstd` library without any need to open Visual Studio solution. Modified: vendor/zstd/dist/appveyor.yml ============================================================================== --- vendor/zstd/dist/appveyor.yml Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/appveyor.yml Mon Oct 22 19:50:43 2018 (r339612) @@ -2,14 +2,13 @@ version: 1.0.{build} branches: only: - - dev - master environment: matrix: - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-dll fullbench-lib" + SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib" ARTIFACT: "true" BUILD: "true" - COMPILER: "gcc" @@ -80,12 +79,22 @@ SET "LDFLAGS=../../zlib/libz.a" && sh -c "%SCRIPT%" && ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true] + ECHO Creating artifacts && + ECHO %cd% && lib\dll\example\build_package.bat && make -C programs DEBUGFLAGS= clean zstd && cd programs\ && 7z a -tzip -mx9 zstd-win-binary-%PLATFORM%.zip zstd.exe && appveyor PushArtifact zstd-win-binary-%PLATFORM%.zip && cp zstd.exe ..\bin\zstd.exe && - cd ..\bin\ && 7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * && + git clone --depth 1 --branch master https://github.com/facebook/zstd && + cd zstd && + git archive --format=tar master -o zstd-src.tar && + ..\zstd -19 zstd-src.tar && + appveyor PushArtifact zstd-src.tar.zst && + certUtil -hashfile zstd-src.tar.zst SHA256 > zstd-src.tar.zst.sha256.sig && + appveyor PushArtifact zstd-src.tar.zst.sha256.sig && + cd ..\..\bin\ && + 7z a -tzip -mx9 zstd-win-release-%PLATFORM%.zip * && appveyor PushArtifact zstd-win-release-%PLATFORM%.zip ) ) Modified: vendor/zstd/dist/contrib/adaptive-compression/adapt.c ============================================================================== --- vendor/zstd/dist/contrib/adaptive-compression/adapt.c Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/contrib/adaptive-compression/adapt.c Mon Oct 22 19:50:43 2018 (r339612) @@ -40,7 +40,6 @@ static unsigned g_compressionLevel = DEFAULT_COMPRESSI static UTIL_time_t g_startTime; static size_t g_streamedSize = 0; static unsigned g_useProgressBar = 1; -static UTIL_freq_t g_ticksPerSecond; static unsigned g_forceCompressionLevel = 0; static unsigned g_minCLevel = 1; static unsigned g_maxCLevel; @@ -576,13 +575,12 @@ static void* compressionThread(void* arg) /* begin compression */ { size_t const useDictSize = MIN(getUseableDictSize(cLevel), job->dictSize); - size_t const dictModeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceRawDict, 1); ZSTD_parameters params = ZSTD_getParams(cLevel, 0, useDictSize); params.cParams.windowLog = 23; { size_t const initError = ZSTD_compressBegin_advanced(ctx->cctx, job->src.start + job->dictSize - useDictSize, useDictSize, params, 0); - size_t const windowSizeError = ZSTD_setCCtxParameter(ctx->cctx, ZSTD_p_forceWindow, 1); - if (ZSTD_isError(dictModeError) || ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) { + size_t const windowSizeError = ZSTD_CCtx_setParameter(ctx->cctx, ZSTD_p_forceMaxWindow, 1); + if (ZSTD_isError(initError) || ZSTD_isError(windowSizeError)) { DISPLAY("Error: something went wrong while starting compression\n"); signalErrorToThreads(ctx); return arg; @@ -644,21 +642,17 @@ static void* compressionThread(void* arg) static void displayProgress(unsigned cLevel, unsigned last) { - UTIL_time_t currTime; - UTIL_getTime(&currTime); + UTIL_time_t currTime = UTIL_getTime(); if (!g_useProgressBar) return; - { - double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_ticksPerSecond, g_startTime, currTime) / 1000.0); + { double const timeElapsed = (double)(UTIL_getSpanTimeMicro(g_startTime, currTime) / 1000.0); double const sizeMB = (double)g_streamedSize / (1 << 20); double const avgCompRate = sizeMB * 1000 / timeElapsed; fprintf(stderr, "\r| Comp. Level: %2u | Time Elapsed: %7.2f s | Data Size: %7.1f MB | Avg Comp. Rate: %6.2f MB/s |", cLevel, timeElapsed/1000.0, sizeMB, avgCompRate); if (last) { fprintf(stderr, "\n"); - } - else { + } else { fflush(stderr); - } - } + } } } static void* outputThread(void* arg) @@ -971,7 +965,6 @@ static int compressFilename(const char* const srcFilen { int ret = 0; fcResources fcr = createFileCompressionResources(srcFilename, dstFilenameOrNull); - UTIL_getTime(&g_startTime); g_streamedSize = 0; ret |= performCompression(fcr.ctx, fcr.srcFile, fcr.otArg); ret |= freeFileCompressionResources(&fcr); @@ -1043,8 +1036,6 @@ int main(int argCount, const char* argv[]) int argNum; filenameTable[0] = stdinmark; g_maxCLevel = ZSTD_maxCLevel(); - - UTIL_initTimer(&g_ticksPerSecond); if (filenameTable == NULL) { DISPLAY("Error: could not allocate sapce for filename table.\n"); Modified: vendor/zstd/dist/contrib/gen_html/Makefile ============================================================================== --- vendor/zstd/dist/contrib/gen_html/Makefile Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/contrib/gen_html/Makefile Mon Oct 22 19:50:43 2018 (r339612) @@ -7,10 +7,10 @@ # in the COPYING file in the root directory of this source tree). # ################################################################ -CFLAGS ?= -O3 -CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment -CFLAGS += $(MOREFLAGS) -FLAGS = $(CPPFLAGS) $(CFLAGS) $(CXXFLAGS) $(LDFLAGS) +CXXFLAGS ?= -O3 +CXXFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 -Wswitch-enum -Wno-comment +CXXFLAGS += $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CXXFLAGS) $(CXXFLAGS) $(LDFLAGS) ZSTDAPI = ../../lib/zstd.h ZSTDMANUAL = ../../doc/zstd_manual.html Modified: vendor/zstd/dist/contrib/meson/meson.build ============================================================================== --- vendor/zstd/dist/contrib/meson/meson.build Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/contrib/meson/meson.build Mon Oct 22 19:50:43 2018 (r339612) @@ -38,21 +38,45 @@ libzstd_srcs = [ libzstd_includes = [include_directories(common_dir, dictbuilder_dir, compress_dir, lib_dir)] -if get_option('legacy_support') - message('Enabling legacy support') - libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=4'] +legacy = get_option('legacy_support') +if legacy == '0' + legacy = 'false' +endif +if legacy != 'false' + if legacy == 'true' + legacy = '1' + endif + #See ZSTD_LEGACY_SUPPORT of programs/README.md + message('Enabling legacy support back to version 0.' + legacy) + legacy_int = legacy.to_int() + if legacy_int > 7 + legacy_int = 7 + endif + libzstd_cflags = ['-DZSTD_LEGACY_SUPPORT=' + legacy] legacy_dir = join_paths(lib_dir, 'legacy') libzstd_includes += [include_directories(legacy_dir)] - libzstd_srcs += [ - join_paths(legacy_dir, 'zstd_v01.c'), - join_paths(legacy_dir, 'zstd_v02.c'), - join_paths(legacy_dir, 'zstd_v03.c'), - join_paths(legacy_dir, 'zstd_v04.c'), - join_paths(legacy_dir, 'zstd_v05.c'), - join_paths(legacy_dir, 'zstd_v06.c'), - join_paths(legacy_dir, 'zstd_v07.c') - ] + if legacy_int <= 1 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v01.c') + endif + if legacy_int <= 2 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v02.c') + endif + if legacy_int <= 3 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v03.c') + endif + if legacy_int <= 4 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v04.c') + endif + if legacy_int <= 5 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v05.c') + endif + if legacy_int <= 6 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v06.c') + endif + if legacy_int <= 7 + libzstd_srcs += join_paths(legacy_dir, 'zstd_v07.c') + endif else libzstd_cflags = [] endif @@ -70,7 +94,9 @@ libzstd = library('zstd', include_directories: libzstd_includes, c_args: libzstd_cflags, dependencies: libzstd_deps, - install: true) + install: true, + soversion: '1', + ) programs_dir = join_paths('..', '..', 'programs') Modified: vendor/zstd/dist/contrib/meson/meson_options.txt ============================================================================== --- vendor/zstd/dist/contrib/meson/meson_options.txt Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/contrib/meson/meson_options.txt Mon Oct 22 19:50:43 2018 (r339612) @@ -1,2 +1,3 @@ option('multithread', type: 'boolean', value: false) -option('legacy_support', type: 'boolean', value: false) +option('legacy_support', type: 'string', value: '4', + description: 'True or false, or 7 to 1 for v0.7+ to v0.1+.') Modified: vendor/zstd/dist/contrib/seekable_format/zstdseek_compress.c ============================================================================== --- vendor/zstd/dist/contrib/seekable_format/zstdseek_compress.c Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/contrib/seekable_format/zstdseek_compress.c Mon Oct 22 19:50:43 2018 (r339612) @@ -147,7 +147,7 @@ size_t ZSTD_seekable_initCStream(ZSTD_seekable_CStream /* make sure maxFrameSize has a reasonable value */ if (maxFrameSize > ZSTD_SEEKABLE_MAX_FRAME_DECOMPRESSED_SIZE) { - return ERROR(compressionParameter_unsupported); + return ERROR(frameParameter_unsupported); } zcs->maxFrameSize = maxFrameSize Modified: vendor/zstd/dist/contrib/seekable_format/zstdseek_decompress.c ============================================================================== --- vendor/zstd/dist/contrib/seekable_format/zstdseek_decompress.c Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/contrib/seekable_format/zstdseek_decompress.c Mon Oct 22 19:50:43 2018 (r339612) @@ -125,7 +125,7 @@ static int ZSTD_seekable_seek_buff(void* opaque, S64 o newOffset = (unsigned long long)buff->size - offset; break; } - if (newOffset < 0 || newOffset > buff->size) { + if (newOffset > buff->size) { return -1; } buff->pos = newOffset; @@ -145,7 +145,7 @@ typedef struct { int checksumFlag; } seekTable_t; -#define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_ABSOLUTEMAX +#define SEEKABLE_BUFF_SIZE ZSTD_BLOCKSIZE_MAX struct ZSTD_seekable_s { ZSTD_DStream* dstream; Modified: vendor/zstd/dist/doc/README.md ============================================================================== --- vendor/zstd/dist/doc/README.md Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/doc/README.md Mon Oct 22 19:50:43 2018 (r339612) @@ -2,19 +2,24 @@ Zstandard Documentation ======================= This directory contains material defining the Zstandard format, -as well as for help using the `zstd` library. +as well as detailed instructions to use `zstd` library. +__`zstd_manual.html`__ : Documentation of `zstd.h` API, in html format. +Click on this link: [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) +to display documentation of latest release in readable format within a browser. + __`zstd_compression_format.md`__ : This document defines the Zstandard compression format. Compliant decoders must adhere to this document, and compliant encoders must generate data that follows it. +Should you look for ressources to develop your own port of Zstandard algorithm, +you may find the following ressources useful : + __`educational_decoder`__ : This directory contains an implementation of a Zstandard decoder, compliant with the Zstandard compression format. It can be used, for example, to better understand the format, -or as the basis for a separate implementation a Zstandard decoder/encoder. +or as the basis for a separate implementation of Zstandard decoder. -__`zstd_manual.html`__ : Documentation on the functions found in `zstd.h`. -See [http://zstd.net/zstd_manual.html](http://zstd.net/zstd_manual.html) for -the manual released with the latest official `zstd` release. - - +[__`decode_corpus`__](https://github.com/facebook/zstd/tree/dev/tests#decodecorpus---tool-to-generate-zstandard-frames-for-decoder-testing) : +This tool, stored in `/tests` directory, is able to generate random valid frames, +which is useful if you wish to test your decoder and verify it fully supports the specification. Added: vendor/zstd/dist/doc/images/CSpeed2.png ============================================================================== Binary file. No diff available. Added: vendor/zstd/dist/doc/images/DSpeed3.png ============================================================================== Binary file. No diff available. Modified: vendor/zstd/dist/doc/images/dict-cr.png ============================================================================== Binary file (source and/or target). No diff available. Modified: vendor/zstd/dist/doc/images/dict-cs.png ============================================================================== Binary file (source and/or target). No diff available. Modified: vendor/zstd/dist/doc/images/dict-ds.png ============================================================================== Binary file (source and/or target). No diff available. Added: vendor/zstd/dist/doc/images/linux-4.7-12-mt-compress.png ============================================================================== Binary file. No diff available. Added: vendor/zstd/dist/doc/images/linux-git-mt-compress.png ============================================================================== Binary file. No diff available. Modified: vendor/zstd/dist/doc/zstd_compression_format.md ============================================================================== --- vendor/zstd/dist/doc/zstd_compression_format.md Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/doc/zstd_compression_format.md Mon Oct 22 19:50:43 2018 (r339612) @@ -257,7 +257,7 @@ a decoder is allowed to reject a compressed frame which requests a memory size beyond decoder's authorized range. For improved interoperability, -decoders are recommended to be compatible with `Window_Size >= 8 MB`, +decoders are recommended to be compatible with `Window_Size <= 8 MB`, and encoders are recommended to not request more than 8 MB. It's merely a recommendation though, decoders are free to support larger or lower limits, Modified: vendor/zstd/dist/doc/zstd_manual.html ============================================================================== --- vendor/zstd/dist/doc/zstd_manual.html Mon Oct 22 19:46:35 2018 (r339611) +++ vendor/zstd/dist/doc/zstd_manual.html Mon Oct 22 19:50:43 2018 (r339612) @@ -1,17 +1,17 @@ <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"> -<title>zstd 1.3.3 Manual</title> +<title>zstd 1.3.4 Manual</title> </head> <body> -<h1>zstd 1.3.3 Manual</h1> +<h1>zstd 1.3.4 Manual</h1> <hr> <a name="Contents"></a><h2>Contents</h2> <ol> <li><a href="#Chapter1">Introduction</a></li> <li><a href="#Chapter2">Version</a></li> <li><a href="#Chapter3">Simple API</a></li> -<li><a href="#Chapter4">Explicit memory management</a></li> +<li><a href="#Chapter4">Explicit context</a></li> <li><a href="#Chapter5">Simple dictionary API</a></li> <li><a href="#Chapter6">Bulk processing dictionary API</a></li> <li><a href="#Chapter7">Streaming</a></li> @@ -19,17 +19,16 @@ <li><a href="#Chapter9">Streaming decompression - HowTo</a></li> <li><a href="#Chapter10">START OF ADVANCED AND EXPERIMENTAL FUNCTIONS</a></li> <li><a href="#Chapter11">Advanced types</a></li> -<li><a href="#Chapter12">Custom memory allocation functions</a></li> -<li><a href="#Chapter13">Frame size functions</a></li> -<li><a href="#Chapter14">Context memory usage</a></li> -<li><a href="#Chapter15">Advanced compression functions</a></li> -<li><a href="#Chapter16">Advanced decompression functions</a></li> -<li><a href="#Chapter17">Advanced streaming functions</a></li> -<li><a href="#Chapter18">Buffer-less and synchronous inner streaming functions</a></li> -<li><a href="#Chapter19">Buffer-less streaming compression (synchronous mode)</a></li> -<li><a href="#Chapter20">Buffer-less streaming decompression (synchronous mode)</a></li> -<li><a href="#Chapter21">New advanced API (experimental)</a></li> -<li><a href="#Chapter22">Block level API</a></li> +<li><a href="#Chapter12">Frame size functions</a></li> +<li><a href="#Chapter13">Memory management</a></li> +<li><a href="#Chapter14">Advanced compression functions</a></li> +<li><a href="#Chapter15">Advanced decompression functions</a></li> +<li><a href="#Chapter16">Advanced streaming functions</a></li> +<li><a href="#Chapter17">Buffer-less and synchronous inner streaming functions</a></li> +<li><a href="#Chapter18">Buffer-less streaming compression (synchronous mode)</a></li> +<li><a href="#Chapter19">Buffer-less streaming decompression (synchronous mode)</a></li> +<li><a href="#Chapter20">New advanced API (experimental)</a></li> +<li><a href="#Chapter21">Block level API</a></li> </ol> <hr> <a name="Chapter1"></a><h2>Introduction</h2><pre> @@ -40,11 +39,11 @@ Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory. Compression can be done in: - a single step (described as Simple API) - - a single step, reusing a context (described as Explicit memory management) + - a single step, reusing a context (described as Explicit context) - unbounded multiple steps (described as Streaming compression) The compression ratio achievable on small data can be highly improved using a dictionary in: - a single step (described as Simple dictionary API) - - a single step, reusing a dictionary (described as Fast dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing dictionary API) Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. Advanced experimental APIs shall never be used with a dynamic library. @@ -103,22 +102,20 @@ unsigned long long ZSTD_getFrameContentSize(const void <pre><b>unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); </b><p> NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). - Both functions work the same way, - but ZSTD_getDecompressedSize() blends - "empty", "unknown" and "error" results in the same return value (0), - while ZSTD_getFrameContentSize() distinguishes them. - - 'src' is the start of a zstd compressed frame. - @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. + Both functions work the same way, but ZSTD_getDecompressedSize() blends + "empty", "unknown" and "error" results to the same return value (0), + while ZSTD_getFrameContentSize() gives them separate return values. + `src` is the start of a zstd compressed frame. + @return : content size to be decompressed, as a 64-bits value _if known and not empty_, 0 otherwise. </p></pre><BR> <h3>Helper functions</h3><pre></pre><b><pre>#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) </b>/* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */<b> -size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case scenario */<b> +size_t ZSTD_compressBound(size_t srcSize); </b>/*!< maximum compressed size in worst case single-pass scenario */<b> unsigned ZSTD_isError(size_t code); </b>/*!< tells if a `size_t` function result is an error code */<b> const char* ZSTD_getErrorName(size_t code); </b>/*!< provides readable string from an error code */<b> int ZSTD_maxCLevel(void); </b>/*!< maximum compression level available */<b> </pre></b><BR> -<a name="Chapter4"></a><h2>Explicit memory management</h2><pre></pre> +<a name="Chapter4"></a><h2>Explicit context</h2><pre></pre> <h3>Compression context</h3><pre> When compressing many times, it is recommended to allocate a context just once, and re-use it for each successive compression operation. @@ -347,11 +344,18 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o ZSTD_frameParameters fParams; } ZSTD_parameters; </b></pre><BR> -<a name="Chapter12"></a><h2>Custom memory allocation functions</h2><pre></pre> - -<pre><b>typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +<pre><b>typedef enum { + ZSTD_dct_auto=0, </b>/* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */<b> + ZSTD_dct_rawContent, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b> + ZSTD_dct_fullDict </b>/* refuses to load a dictionary if it does not respect Zstandard's specification */<b> +} ZSTD_dictContentType_e; </b></pre><BR> -<a name="Chapter13"></a><h2>Frame size functions</h2><pre></pre> +<pre><b>typedef enum { + ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b> + ZSTD_dlm_byRef, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b> +} ZSTD_dictLoadMethod_e; +</b></pre><BR> +<a name="Chapter12"></a><h2>Frame size functions</h2><pre></pre> <pre><b>size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); </b><p> `src` should point to the start of a ZSTD encoded frame or skippable frame @@ -390,7 +394,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_o @return : size of the Frame Header </p></pre><BR> -<a name="Chapter14"></a><h2>Context memory usage</h2><pre></pre> +<a name="Chapter13"></a><h2>Memory management</h2><pre></pre> <pre><b>size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); @@ -399,7 +403,7 @@ size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); </b><p> These functions give the current memory usage of selected object. - Object memory usage can evolve when re-used multiple times. + Object memory usage can evolve when re-used. </p></pre><BR> <pre><b>size_t ZSTD_estimateCCtxSize(int compressionLevel); @@ -412,8 +416,8 @@ size_t ZSTD_estimateDCtxSize(void); It will also consider src size to be arbitrarily "large", which is worst case. If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is > 1. - Note : CCtx estimation is only correct for single-threaded compression + ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1. + Note : CCtx size estimation is only correct for single-threaded compression. </p></pre><BR> <pre><b>size_t ZSTD_estimateCStreamSize(int compressionLevel); @@ -425,8 +429,8 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* It will also consider src size to be arbitrarily "large", which is worst case. If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbThreads is set to a value > 1. - Note : CStream estimation is only correct for single-threaded compression. + ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1. + Note : CStream size estimation is only correct for single-threaded compression. ZSTD_DStream memory budget depends on window Size. This information can be passed manually, using ZSTD_estimateDStreamSize, or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); @@ -435,83 +439,59 @@ size_t ZSTD_estimateDStreamSize_fromFrame(const void* In this case, get total size by adding ZSTD_estimate?DictSize </p></pre><BR> -<pre><b>typedef enum { - ZSTD_dlm_byCopy = 0, </b>/**< Copy dictionary content internally */<b> - ZSTD_dlm_byRef, </b>/**< Reference dictionary content -- the dictionary buffer must outlive its users. */<b> -} ZSTD_dictLoadMethod_e; -</b></pre><BR> <pre><b>size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); </b><p> ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). - ZSTD_estimateCStreamSize_advanced_usingCParams() makes it possible to control precisely compression parameters, like ZSTD_createCDict_advanced(). - Note : dictionary created by reference using ZSTD_dlm_byRef are smaller + ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. </p></pre><BR> -<a name="Chapter15"></a><h2>Advanced compression functions</h2><pre></pre> - -<pre><b>ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); -</b><p> Create a ZSTD compression context using external alloc and free functions +<pre><b>ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b> +</b><p> Initialize an object using a pre-allocated fixed-size buffer. + workspace: The memory area to emplace the object into. + Provided pointer *must be 8-bytes aligned*. + Buffer must outlive object. + workspaceSize: Use ZSTD_estimate*Size() to determine + how large workspace must be to support target scenario. + @return : pointer to object (same address as workspace, just different type), + or NULL if error (size too small, incorrect alignment, etc.) + Note : zstd will never resize nor malloc() when using a static buffer. + If the object requires more memory than available, + zstd will just error out (typically ZSTD_error_memory_allocation). + Note 2 : there is no corresponding "free" function. + Since workspace is allocated externally, it must be freed externally too. + Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + into its associated cParams. + Limitation 1 : currently not compatible with internal dictionary creation, triggered by + ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + Limitation 2 : static cctx currently not compatible with multi-threading. + Limitation 3 : static dctx is incompatible with legacy support. + </p></pre><BR> -<pre><b>ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); -</b><p> workspace: The memory area to emplace the context into. - Provided pointer must 8-bytes aligned. - It must outlive context usage. - workspaceSize: Use ZSTD_estimateCCtxSize() or ZSTD_estimateCStreamSize() - to determine how large workspace must be to support scenario. - @return : pointer to ZSTD_CCtx* (same address as workspace, but different type), - or NULL if error (typically size too small) - Note : zstd will never resize nor malloc() when using a static cctx. - If it needs more memory than available, it will simply error out. - Note 2 : there is no corresponding "free" function. - Since workspace was allocated externally, it must be freed externally too. - Limitation 1 : currently not compatible with internal CDict creation, such as - ZSTD_CCtx_loadDictionary() or ZSTD_initCStream_usingDict(). - Limitation 2 : currently not compatible with multi-threading +<pre><b>ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticDCtx() */<b> +</b></pre><BR> +<pre><b>typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; </b>/**< this constant defers to stdlib's functions */<b> +</b><p> These prototypes make it possible to pass your own allocation/free functions. + ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + All allocation/free operations will be completed using these custom variants instead of regular <stdlib.h> ones. </p></pre><BR> +<a name="Chapter14"></a><h2>Advanced compression functions</h2><pre></pre> + <pre><b>ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); </b><p> Create a digested dictionary for compression Dictionary content is simply referenced, and therefore stays in dictBuffer. It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict </p></pre><BR> -<pre><b>typedef enum { ZSTD_dm_auto=0, </b>/* dictionary is "full" if it starts with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */<b> - ZSTD_dm_rawContent, </b>/* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */<b> - ZSTD_dm_fullDict </b>/* refuses to load a dictionary if it does not respect Zstandard's specification */<b> -} ZSTD_dictMode_e; -</b></pre><BR> -<pre><b>ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictMode_e dictMode, - ZSTD_compressionParameters cParams, - ZSTD_customMem customMem); -</b><p> Create a ZSTD_CDict using external alloc and free, and customized compression parameters -</p></pre><BR> - -<pre><b>ZSTD_CDict* ZSTD_initStaticCDict( - void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictMode_e dictMode, - ZSTD_compressionParameters cParams); -</b><p> Generate a digested dictionary in provided memory area. - workspace: The memory area to emplace the dictionary into. - Provided pointer must 8-bytes aligned. - It must outlive dictionary usage. - workspaceSize: Use ZSTD_estimateCDictSize() - to determine how large workspace must be. - cParams : use ZSTD_getCParams() to transform a compression level - into its relevants cParams. - @return : pointer to ZSTD_CDict* (same address as workspace, but different type), - or NULL if error (typically, size too small). - Note : there is no corresponding "free" function. - Since workspace was allocated externally, it must be freed externally. - -</p></pre><BR> - <pre><b>ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); </b><p> @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. `estimatedSrcSize` value is optional, select 0 if not known @@ -546,7 +526,7 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_di </b><p> Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters </p></pre><BR> -<a name="Chapter16"></a><h2>Advanced decompression functions</h2><pre></pre> +<a name="Chapter15"></a><h2>Advanced decompression functions</h2><pre></pre> <pre><b>unsigned ZSTD_isFrame(const void* buffer, size_t size); </b><p> Tells if the content of `buffer` starts with a valid Frame Identifier. @@ -555,28 +535,6 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_di Note 3 : Skippable Frame Identifiers are considered valid. </p></pre><BR> -<pre><b>ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -</b><p> Create a ZSTD decompression context using external alloc and free functions -</p></pre><BR> - -<pre><b>ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); -</b><p> workspace: The memory area to emplace the context into. - Provided pointer must 8-bytes aligned. - It must outlive context usage. - workspaceSize: Use ZSTD_estimateDCtxSize() or ZSTD_estimateDStreamSize() - to determine how large workspace must be to support scenario. - @return : pointer to ZSTD_DCtx* (same address as workspace, but different type), - or NULL if error (typically size too small) - Note : zstd will never resize nor malloc() when using a static dctx. - If it needs more memory than available, it will simply error out. - Note 2 : static dctx is incompatible with legacy support - Note 3 : there is no corresponding "free" function. - Since workspace was allocated externally, it must be freed externally. - Limitation : currently not compatible with internal DDict creation, - such as ZSTD_initDStream_usingDict(). - -</p></pre><BR> - <pre><b>ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); </b><p> Create a digested dictionary, ready to start decompression operation without startup delay. Dictionary content is referenced, and therefore stays in dictBuffer. @@ -584,27 +542,6 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_di it must remain read accessible throughout the lifetime of DDict </p></pre><BR> -<pre><b>ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_customMem customMem); -</b><p> Create a ZSTD_DDict using external alloc and free, optionally by reference -</p></pre><BR> - -<pre><b>ZSTD_DDict* ZSTD_initStaticDDict(void* workspace, size_t workspaceSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod); -</b><p> Generate a digested dictionary in provided memory area. - workspace: The memory area to emplace the dictionary into. - Provided pointer must 8-bytes aligned. - It must outlive dictionary usage. - workspaceSize: Use ZSTD_estimateDDictSize() - to determine how large workspace must be. - @return : pointer to ZSTD_DDict*, or NULL if error (size too small) - Note : there is no corresponding "free" function. - Since workspace was allocated externally, it must be freed externally. - -</p></pre><BR> - <pre><b>unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); </b><p> Provides the dictID stored within dictionary. if @return == 0, the dictionary is not conformant with Zstandard specification. @@ -629,11 +566,9 @@ size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_di When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. </p></pre><BR> -<a name="Chapter17"></a><h2>Advanced streaming functions</h2><pre></pre> +<a name="Chapter16"></a><h2>Advanced streaming functions</h2><pre></pre> -<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); -ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticCCtx() */<b> -size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b> +<h3>Advanced Streaming compression functions</h3><pre></pre><b><pre>size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If it is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, "0" also disables frame content size field. It may be enabled in the future. */<b> size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); </b>/**< creates of an internal CDict (incompatible with static CCtx), except if dict == NULL or dictSize < 8, in which case no dict is used. Note: dict is loaded with ZSTD_dm_auto (treated as a full zstd dictionary if it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy.*/<b> size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); </b>/**< pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. dict is loaded with ZSTD_dm_auto and ZSTD_dlm_byCopy. */<b> @@ -647,26 +582,30 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStre If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, - but it may change to mean "empty" in some future version, so prefer using macro ZSTD_CONTENTSIZE_UNKNOWN. + but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. @return : 0, or an error code (which can be tested using ZSTD_isError()) </p></pre><BR> -<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); -ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); </b>/**< same as ZSTD_initStaticDCtx() */<b> -typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; +<pre><b>typedef struct { + unsigned long long ingested; + unsigned long long consumed; + unsigned long long produced; +} ZSTD_frameProgression; +</b></pre><BR> +<h3>Advanced Streaming decompression functions</h3><pre></pre><b><pre>typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); </b>/* obsolete : this API will be removed in a future version */<b> size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); </b>/**< note: no dictionary will be used if dict == NULL or dictSize < 8 */<b> size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); </b>/**< note : ddict is referenced, it must outlive decompression session */<b> size_t ZSTD_resetDStream(ZSTD_DStream* zds); </b>/**< re-use decompression parameters from previous init; saves dictionary loading */<b> </pre></b><BR> -<a name="Chapter18"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> +<a name="Chapter17"></a><h2>Buffer-less and synchronous inner streaming functions</h2><pre> This is an advanced API, giving full control over buffer management, for users which need direct control over memory. But it's also a complex one, with several restrictions, documented below. Prefer normal streaming API for an easier experience. <BR></pre> -<a name="Chapter19"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> +<a name="Chapter18"></a><h2>Buffer-less streaming compression (synchronous mode)</h2><pre> A ZSTD_CCtx object is required to track streaming operations. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. ZSTD_CCtx object can be re-used multiple times within successive compression operations. @@ -702,7 +641,7 @@ size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); </b>/* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */<b> size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); </b>/**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */<b> </pre></b><BR> -<a name="Chapter20"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> +<a name="Chapter19"></a><h2>Buffer-less streaming decompression (synchronous mode)</h2><pre> A ZSTD_DCtx object is required to track streaming operations. Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. A ZSTD_DCtx object can be re-used multiple times. @@ -788,15 +727,15 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long </pre></b><BR> <pre><b>typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; </b></pre><BR> -<a name="Chapter21"></a><h2>New advanced API (experimental)</h2><pre></pre> +<a name="Chapter20"></a><h2>New advanced API (experimental)</h2><pre></pre> <pre><b>typedef enum { - </b>/* Question : should we have a format ZSTD_f_auto ?<b> - * For the time being, it would mean exactly the same as ZSTD_f_zstd1. - * But, in the future, should several formats be supported, + </b>/* Opened question : should we have a format ZSTD_f_auto ?<b> + * Today, it would mean exactly the same as ZSTD_f_zstd1. + * But, in the future, should several formats become supported, * on the compression side, it would mean "default format". - * On the decompression side, it would mean "multi format", - * and ZSTD_f_zstd1 could be reserved to mean "accept *only* zstd frames". + * On the decompression side, it would mean "automatic format detection", + * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". * Since meaning is a little different, another option could be to define different enums for compression and decompression. * This question could be kept for later, when there are actually multiple formats to support, * but there is also the question of pinning enum values, and pinning value `0` is especially important */ @@ -814,43 +753,77 @@ size_t ZSTD_decodingBufferSize_min(unsigned long long </b>/* compression parameters */<b> ZSTD_p_compressionLevel=100, </b>/* Update all compression parameters according to pre-defined cLevel table<b> * Default level is ZSTD_CLEVEL_DEFAULT==3. - * Special: value 0 means "do not change cLevel". */ + * Special: value 0 means "do not change cLevel". + * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type. + * Note 2 : setting a level sets all default values of other compression parameters. + * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ ZSTD_p_windowLog, </b>/* Maximum allowed back-reference distance, expressed as power of 2.<b> * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "do not change windowLog". + * Special: value 0 means "use default windowLog". * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27) - * requires setting the maximum window size at least as large during decompression. */ + * requires explicitly allowing such window size during decompression stage. */ ZSTD_p_hashLog, </b>/* Size of the probe table, as a power of 2.<b> * Resulting table size is (1 << (hashLog+2)). * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. * Larger tables improve compression ratio of strategies <= dFast, * and improve speed of strategies > dFast. - * Special: value 0 means "do not change hashLog". */ + * Special: value 0 means "use default hashLog". */ ZSTD_p_chainLog, </b>/* Size of the full-search table, as a power of 2.<b> * Resulting table size is (1 << (chainLog+2)). * Larger tables result in better and slower compression. * This parameter is useless when using "fast" strategy. - * Special: value 0 means "do not change chainLog". */ + * Special: value 0 means "use default chainLog". */ ZSTD_p_searchLog, </b>/* Number of search attempts, as a power of 2.<b> * More attempts result in better and slower compression. * This parameter is useless when using "fast" and "dFast" strategies. - * Special: value 0 means "do not change searchLog". */ + * Special: value 0 means "use default searchLog". */ ZSTD_p_minMatch, </b>/* Minimum size of searched matches (note : repCode matches can be smaller).<b> * Larger values make faster compression and decompression, but decrease ratio. * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX. * Note that currently, for all strategies < btopt, effective minimum is 4. - * Note that currently, for all strategies > fast, effective maximum is 6. - * Special: value 0 means "do not change minMatchLength". */ - ZSTD_p_targetLength, </b>/* Only useful for strategies >= btopt.<b> - * Length of Match considered "good enough" to stop search. - * Larger values make compression stronger and slower. - * Special: value 0 means "do not change targetLength". */ + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_p_targetLength, </b>/* Impact of this field depends on strategy.<b> + * For strategies btopt & btultra: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ ZSTD_p_compressionStrategy, </b>/* See ZSTD_strategy enum definition.<b> * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. * The higher the value of selected strategy, the more complex it is, * resulting in stronger and slower compression. - * Special: value 0 means "do not change strategy". */ + * Special: value 0 means "use default strategy". */ + ZSTD_p_enableLongDistanceMatching=160, </b>/* Enable long distance matching.<b> + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB + * except when expressly set to a different value. */ + ZSTD_p_ldmHashLog, </b>/* Size of the table for long distance matching, as a power of 2.<b> + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_p_ldmMinMatch, </b>/* Minimum match size for long distance matcher.<b> + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_p_ldmBucketSizeLog, </b>/* Log size of each bucket in the LDM hash table for collision resolution.<b> + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX . + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_p_ldmHashEveryLog, </b>/* Frequency of inserting/looking up entries in the LDM hash table.<b> + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashEveryLog". */ *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201810221950.w9MJohXr076158>