Date: Sat, 6 May 2017 10:26:40 +0000 (UTC) From: Baptiste Daroussin <bapt@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r317877 - in head: contrib/zstd contrib/zstd/contrib contrib/zstd/contrib/pzstd contrib/zstd/contrib/pzstd/utils/test contrib/zstd/doc contrib/zstd/doc/educational_decoder contrib/zstd/... Message-ID: <201705061026.v46AQesA089962@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: bapt Date: Sat May 6 10:26:40 2017 New Revision: 317877 URL: https://svnweb.freebsd.org/changeset/base/317877 Log: Import Zstandard 1.2.0 Among new things it is now threaded by default, use zstd -T# to chose the number of threads not that -T0 will automatically determine the number of threads based on the number of CPU online. Added: head/contrib/zstd/contrib/cleanTabs - copied unchanged from r317876, vendor/zstd/dist/contrib/cleanTabs head/contrib/zstd/programs/zstd.1.md - copied unchanged from r317876, vendor/zstd/dist/programs/zstd.1.md Deleted: head/contrib/zstd/.gitignore head/contrib/zstd/.travis.yml Modified: head/contrib/zstd/Makefile head/contrib/zstd/NEWS head/contrib/zstd/README.md head/contrib/zstd/appveyor.yml head/contrib/zstd/circle.yml head/contrib/zstd/contrib/pzstd/Options.cpp head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp head/contrib/zstd/doc/educational_decoder/zstd_decompress.c head/contrib/zstd/doc/images/Cspeed4.png head/contrib/zstd/doc/images/Dspeed4.png head/contrib/zstd/doc/images/dict-cr.png head/contrib/zstd/doc/images/dict-cs.png head/contrib/zstd/doc/images/dict-ds.png head/contrib/zstd/doc/zstd_compression_format.md head/contrib/zstd/doc/zstd_manual.html head/contrib/zstd/examples/simple_compression.c head/contrib/zstd/examples/streaming_compression.c head/contrib/zstd/examples/streaming_decompression.c head/contrib/zstd/lib/Makefile head/contrib/zstd/lib/README.md head/contrib/zstd/lib/common/bitstream.h head/contrib/zstd/lib/common/error_private.c head/contrib/zstd/lib/common/fse.h head/contrib/zstd/lib/common/huf.h head/contrib/zstd/lib/common/mem.h head/contrib/zstd/lib/common/zstd_errors.h head/contrib/zstd/lib/common/zstd_internal.h head/contrib/zstd/lib/compress/fse_compress.c head/contrib/zstd/lib/compress/zstd_compress.c head/contrib/zstd/lib/compress/zstd_opt.h head/contrib/zstd/lib/compress/zstdmt_compress.c head/contrib/zstd/lib/decompress/zstd_decompress.c head/contrib/zstd/lib/dictBuilder/cover.c head/contrib/zstd/lib/dictBuilder/zdict.c head/contrib/zstd/lib/dictBuilder/zdict.h head/contrib/zstd/lib/legacy/zstd_v01.c head/contrib/zstd/lib/legacy/zstd_v02.c head/contrib/zstd/lib/legacy/zstd_v03.c head/contrib/zstd/lib/legacy/zstd_v04.c head/contrib/zstd/lib/legacy/zstd_v05.c head/contrib/zstd/lib/legacy/zstd_v06.c head/contrib/zstd/lib/zstd.h head/contrib/zstd/programs/Makefile head/contrib/zstd/programs/README.md head/contrib/zstd/programs/bench.c head/contrib/zstd/programs/dibio.c head/contrib/zstd/programs/fileio.c head/contrib/zstd/programs/fileio.h head/contrib/zstd/programs/platform.h head/contrib/zstd/programs/util.h head/contrib/zstd/programs/zstd.1 head/contrib/zstd/programs/zstdcli.c head/contrib/zstd/tests/Makefile head/contrib/zstd/tests/decodecorpus.c head/contrib/zstd/tests/fullbench.c head/contrib/zstd/tests/fuzzer.c head/contrib/zstd/tests/paramgrill.c head/contrib/zstd/tests/playTests.sh head/contrib/zstd/tests/test-zstd-speed.py head/contrib/zstd/tests/zbufftest.c head/contrib/zstd/tests/zstreamtest.c head/contrib/zstd/zlibWrapper/examples/zwrapbench.c head/usr.bin/zstd/Makefile Directory Properties: head/contrib/zstd/ (props changed) Modified: head/contrib/zstd/Makefile ============================================================================== --- head/contrib/zstd/Makefile Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/Makefile Sat May 6 10:26:40 2017 (r317877) @@ -90,6 +90,10 @@ examples: manual: $(MAKE) -C contrib/gen_html $@ +.PHONY: cleanTabs +cleanTabs: + cd contrib; ./cleanTabs + .PHONY: clean clean: @$(MAKE) -C $(ZSTDDIR) $@ > $(VOID) @@ -105,9 +109,15 @@ clean: # make install is validated only for Linux, OSX, Hurd and some BSD targets #------------------------------------------------------------------------------ ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) + HOST_OS = POSIX -.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan +CMAKE_PARAMS = -DZSTD_BUILD_CONTRIB:BOOL=ON -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_ZLIB_SUPPORT:BOOL=ON -DZSTD_LZMA_SUPPORT:BOOL=ON +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + +.PHONY: install uninstall travis-install clangtest gpptest armtest usan asan uasan install: @$(MAKE) -C $(ZSTDDIR) $@ @$(MAKE) -C $(PRGDIR) $@ @@ -152,16 +162,16 @@ ppc64build: clean CC=powerpc-linux-gnu-gcc CFLAGS="-m64 -Werror" $(MAKE) allarch armfuzz: clean - CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest aarch64fuzz: clean - CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest ppcfuzz: clean - CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" $(MAKE) -C $(TESTDIR) fuzztest + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static MOREFLAGS="-static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest ppc64fuzz: clean - CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" $(MAKE) -C $(TESTDIR) fuzztest + CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests $(MAKE) -C $(TESTDIR) fuzztest gpptest: clean CC=g++ $(MAKE) -C $(PRGDIR) all CFLAGS="-O3 -Wall -Wextra -Wundef -Wshadow -Wcast-align -Werror" @@ -180,19 +190,19 @@ clangtest: clean armtest: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(TESTDIR) test CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests aarch64test: $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" + $(MAKE) -C $(TESTDIR) test CC=aarch64-linux-gnu-gcc QEMU_SYS=qemu-aarch64-static ZSTDRTTEST= MOREFLAGS="-Werror -static" FUZZER_FLAGS=--no-big-tests ppctest: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc-static ZSTDRTTEST= MOREFLAGS="-Werror -Wno-attributes -static" FUZZER_FLAGS=--no-big-tests ppc64test: clean $(MAKE) -C $(TESTDIR) datagen # use native, faster - $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" + $(MAKE) -C $(TESTDIR) test CC=powerpc-linux-gnu-gcc QEMU_SYS=qemu-ppc64-static ZSTDRTTEST= MOREFLAGS="-m64 -static" FUZZER_FLAGS=--no-big-tests arm-ppc-compilation: $(MAKE) -C $(PRGDIR) clean zstd CC=arm-linux-gnueabi-gcc QEMU_SYS=qemu-arm-static ZSTDRTTEST= MOREFLAGS="-Werror -static" @@ -263,7 +273,7 @@ endif ifneq (,$(filter MSYS%,$(shell uname))) HOST_OS = MSYS -CMAKE_PARAMS = -G"MSYS Makefiles" +CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON endif @@ -275,7 +285,7 @@ cmakebuild: cmake --version $(RM) -r $(BUILDIR)/cmake/build mkdir $(BUILDIR)/cmake/build - cd $(BUILDIR)/cmake/build ; cmake -DPREFIX:STRING=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall c90build: clean gcc -v Modified: head/contrib/zstd/NEWS ============================================================================== --- head/contrib/zstd/NEWS Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/NEWS Sat May 6 10:26:40 2017 (r317877) @@ -1,3 +1,25 @@ +v1.2.0 +cli : changed : Multithreading enabled by default (use target zstd-nomt or HAVE_THREAD=0 to disable) +cli : new : command -T0 means "detect and use nb of cores", by Sean Purcell +cli : new : zstdmt symlink hardwired to `zstd -T0` +cli : new : command --threads=# (#671) +cli : changed : cover dictionary builder by default, for improved quality, by Nick Terrell +cli : new : commands --train-cover and --train-legacy, to select dictionary algorithm and parameters +cli : experimental targets `zstd4` and `xzstd4`, with support for lz4 format, by Sean Purcell +cli : fix : does not output compressed data on console +cli : fix : ignore symbolic links unless --force specified, +API : breaking change : ZSTD_createCDict_advanced(), only use compressionParameters as argument +API : added : prototypes ZSTD_*_usingCDict_advanced(), for direct control over frameParameters. +API : improved: ZSTDMT_compressCCtx() reduced memory usage +API : fix : ZSTDMT_compressCCtx() now provides srcSize in header (#634) +API : fix : src size stored in frame header is controlled at end of frame +API : fix : enforced consistent rules for pledgedSrcSize==0 (#641) +API : fix : error code "GENERIC" replaced by "dstSizeTooSmall" when appropriate +build: improved cmake script, by @Majlen +build: enabled Multi-threading support for *BSD, by Baptiste Daroussin +tools: updated Paramgrill. Command -O# provides best parameters for sample and speed target. +new : contrib/linux-kernel version, by Nick Terrell + v1.1.4 cli : new : can compress in *.gz format, using --format=gzip command, by Przemyslaw Skibinski cli : new : advanced benchmark command --priority=rt @@ -5,13 +27,13 @@ cli : fix : write on sparse-enabled file cli : fix : --rm remains silent when input is stdin cli : experimental : xzstd, with support for xz/lzma decoding, by Przemyslaw Skibinski speed : improved decompression speed in streaming mode for single shot scenarios (+5%) -memory : DDict (decompression dictionary) memory usage down from 150 KB to 20 KB -arch : 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell +memory: DDict (decompression dictionary) memory usage down from 150 KB to 20 KB +arch: 32-bits variant able to generate and decode very long matches (>32 MB), by Sean Purcell API : new : ZSTD_findFrameCompressedSize(), ZSTD_getFrameContentSize(), ZSTD_findDecompressedSize() API : changed : dropped support of legacy versions <= v0.3 (can be changed by modifying ZSTD_LEGACY_SUPPORT value) -build: new: meson build system in contrib/meson, by Dima Krasner -build: improved cmake script, by @Majlen -build: added -Wformat-security flag, as recommended by Padraig Brady +build : new: meson build system in contrib/meson, by Dima Krasner +build : improved cmake script, by @Majlen +build : added -Wformat-security flag, as recommended by Padraig Brady doc : new : educational decoder, by Sean Purcell v1.1.3 Modified: head/contrib/zstd/README.md ============================================================================== --- head/contrib/zstd/README.md Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/README.md Sat May 6 10:26:40 2017 (r317877) @@ -12,13 +12,13 @@ you can consult a list of known ports on |dev | [](https://travis-ci.org/facebook/zstd) | As a reference, several fast compression algorithms were tested and compared -on a server running Linux Mint Debian Edition (`Linux version 4.8.0-1-amd64`), +on a server running Linux Debian (`Linux version 4.8.0-1-amd64`), with a Core i7-6700K CPU @ 4.0GHz, -using [lzbench v1.6], an open-source in-memory benchmark by @inikep +using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 6.3.0, on the [Silesia compression corpus]. -[lzbench v1.6]: https://github.com/inikep/lzbench +[lzbench]: https://github.com/inikep/lzbench [Silesia compression corpus]: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia | Compressor name | Ratio | Compression| Decompress.| @@ -38,7 +38,12 @@ on the [Silesia compression corpus]. Zstd can also offer stronger compression ratios at the cost of compression speed. Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, a property shared by most LZ compression algorithms, such as [zlib] or lzma. -The following tests were run on a Core i7-3930K CPU @ 4.5GHz, using [lzbench], an open-source in-memory benchmark by @inikep compiled with GCC 5.2.1, on the [Silesia compression corpus]. +The following tests were run +on a server running Linux Debian (`Linux version 4.8.0-1-amd64`) +with a Core i7-6700K CPU @ 4.0GHz, +using [lzbench], an open-source in-memory benchmark by @inikep +compiled with GCC 6.3.0, +on the [Silesia compression corpus]. Compression Speed vs Ratio | Decompression Speed ---------------------------|-------------------- Modified: head/contrib/zstd/appveyor.yml ============================================================================== --- head/contrib/zstd/appveyor.yml Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/appveyor.yml Sat May 6 10:26:40 2017 (r317877) @@ -1,66 +1,103 @@ -version: 1.0.{build} -environment: - matrix: - - COMPILER: "gcc" - PLATFORM: "mingw64" - MAKE_PARAMS: '"make test && make lib && make -C tests test-symbols fullbench-dll fullbench-lib"' - - COMPILER: "gcc" - PLATFORM: "mingw32" - MAKE_PARAMS: '"make -C tests test-zstd test-fullbench test-fuzzer test-invalidDictionaries"' - - COMPILER: "gcc" - PLATFORM: "clang" - MAKE_PARAMS: '"make -C tests zstd fullbench fuzzer paramgrill datagen CC=clang MOREFLAGS="--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion""' - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Debug" - PLATFORM: "Win32" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "x64" - - COMPILER: "visual" - CONFIGURATION: "Release" - PLATFORM: "Win32" +- + version: 1.0.{build} + branches: + only: + - dev + - master + environment: + matrix: + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "make allarch && make -C tests test-symbols fullbench-dll fullbench-lib" + ARTIFACT: "true" + BUILD: "true" + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x86" + SCRIPT: "make allarch" + ARTIFACT: "true" + BUILD: "true" + - COMPILER: "clang" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch" + BUILD: "true" + + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "" + TEST: "cmake" + + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "" + TEST: "pzstd" + + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Release" -install: + install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - - MKDIR bin - - if [%COMPILER%]==[gcc] SET PATH_ORIGINAL=%PATH% - - if [%COMPILER%]==[gcc] ( - SET "PATH_MINGW32=c:\MinGW\bin;c:\MinGW\usr\bin" && - SET "PATH_MINGW64=c:\msys64\mingw64\bin;c:\msys64\usr\bin" && - COPY C:\msys64\usr\bin\make.exe C:\MinGW\bin\make.exe && - COPY C:\MinGW\bin\gcc.exe C:\MinGW\bin\cc.exe - ) else ( - IF [%PLATFORM%]==[x64] (SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;") + - SET PATH_ORIGINAL=%PATH% + - if [%HOST%]==[mingw] ( + SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && + SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe + ) + - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( + SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + ) + + build_script: + - if [%HOST%]==[mingw] ( + ( if [%PLATFORM%]==[x64] ( + SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%" + ) else if [%PLATFORM%]==[x86] ( + SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%" + ) ) ) - -build_script: - - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% - - if [%PLATFORM%]==[mingw32] SET PATH=%PATH_MINGW32%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[mingw64] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%PLATFORM%]==[clang] SET PATH=%PATH_MINGW64%;%PATH_ORIGINAL% - - if [%COMPILER%]==[gcc] ( - ECHO *** && - ECHO *** Building %PLATFORM% && - ECHO *** && + - if [%HOST%]==[mingw] if [%BUILD%]==[true] ( make -v && - cc -v && - ECHO %MAKE_PARAMS% && - sh -c %MAKE_PARAMS% - ) - - if [%PLATFORM%]==[clang] COPY tests\fuzzer.exe tests\fuzzer_clang.exe - - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw64] ( - COPY programs\zstd.exe bin\zstd.exe && - appveyor PushArtifact bin\zstd.exe - ) - - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[mingw32] ( - COPY programs\zstd.exe bin\zstd32.exe && - appveyor PushArtifact bin\zstd32.exe + sh -c "%COMPILER% -v" && + ECHO Building zlib to static link && + SET "CC=%COMPILER%" && + sh -c "cd .. && git clone --depth 1 --branch v1.2.11 https://github.com/madler/zlib" && + sh -c "cd ../zlib && make -f win32/Makefile.gcc libz.a" + ECHO Building zstd && + SET "CPPFLAGS=-I../../zlib" && + SET "LDFLAGS=../../zlib/libz.a" && + sh -c "%SCRIPT%" && + ( if [%COMPILER%]==[gcc] if [%ARTIFACT%]==[true] + lib\dll\example\build_package.bat && + make -C programs DEBUGFLAGS= clean zstd && + cp programs\zstd.exe zstd_%PLATFORM%.exe && + appveyor PushArtifact zstd_%PLATFORM%.exe && + cp programs\zstd.exe bin\zstd.exe && + make -C programs DEBUGFLAGS= clean zstdmt && + cp programs\zstd.exe bin\zstdmt.exe && + cd bin\ && 7z a -tzip zstd-win-release-%PLATFORM%.zip * && + appveyor PushArtifact zstd-win-release-%PLATFORM%.zip + ) ) - - if [%COMPILER%]==[gcc] make clean - - if [%COMPILER%]==[visual] ( + - if [%HOST%]==[visual] ( ECHO *** && ECHO *** Building Visual Studio 2008 %PLATFORM%\%CONFIGURATION% in %APPVEYOR_BUILD_FOLDER% && ECHO *** && @@ -111,29 +148,26 @@ build_script: COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ ) -test_script: + test_script: - ECHO Testing %COMPILER% %PLATFORM% %CONFIGURATION% - - SET FUZZERTEST=-T1mn - - if [%COMPILER%]==[gcc] if [%PLATFORM%]==[clang] ( - tests\fuzzer_clang.exe %FUZZERTEST% && - ECHO *** && - ECHO *** Building cmake for %PLATFORM% && - ECHO *** && + - SET "CC=gcc" + - SET "CXX=g++" + - if [%TEST%]==[cmake] ( mkdir build\cmake\build && cd build\cmake\build && cmake -G "Visual Studio 14 2015 Win64" .. && cd ..\..\.. && - make clean && - ECHO *** && - ECHO *** Building pzstd for %PLATFORM% && - ECHO *** && + make clean + ) + - if [%TEST%]==[pzstd] ( make -C contrib\pzstd googletest-mingw64 && make -C contrib\pzstd pzstd.exe && make -C contrib\pzstd tests && make -C contrib\pzstd check && make -C contrib\pzstd clean ) - - if [%COMPILER%]==[visual] if [%CONFIGURATION%]==[Release] ( + - SET "FUZZERTEST=-T30s" + - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] ( CD tests && SET ZSTD=./zstd.exe && sh -e playTests.sh --test-large-data && @@ -146,33 +180,76 @@ test_script: fuzzer_VS2015_%PLATFORM%_Release.exe %FUZZERTEST% ) -branches: - only: - - dev - - master - -artifacts: - - path: bin\zstd.exe - - path: bin\zstd32.exe - -deploy: -- provider: GitHub - auth_token: - secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - artifact: bin\zstd.exe - force_update: true - on: - branch: autobuild - COMPILER: gcc - PLATFORM: "mingw64" - appveyor_repo_tag: true -- provider: GitHub - auth_token: - secure: LgJo8emYc3sFnlNWkGl4/VYK3nk/8+RagcsqDlAi3xeqNGNutnKjcftjg84uJoT4 - artifact: bin\zstd32.exe - force_update: true - on: - branch: autobuild - COMPILER: gcc - PLATFORM: "mingw32" - appveyor_repo_tag: true +- + version: 1.0.{build} + environment: + matrix: + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "make allarch" + - COMPILER: "gcc" + HOST: "mingw" + PLATFORM: "x86" + SCRIPT: "make allarch" + - COMPILER: "clang" + HOST: "mingw" + PLATFORM: "x64" + SCRIPT: "MOREFLAGS='--target=x86_64-w64-mingw32 -Werror -Wconversion -Wno-sign-conversion' make allarch" + + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Debug" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "x64" + CONFIGURATION: "Release" + - COMPILER: "visual" + HOST: "visual" + PLATFORM: "Win32" + CONFIGURATION: "Release" + + install: + - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% + - SET PATH_ORIGINAL=%PATH% + - if [%HOST%]==[mingw] ( + SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && + SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin\make.exe && + COPY C:\msys64\usr\bin\make.exe C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin\make.exe + ) + - IF [%HOST%]==[visual] IF [%PLATFORM%]==[x64] ( + SET ADDITIONALPARAM=/p:LibraryPath="C:\Program Files\Microsoft SDKs\Windows\v7.1\lib\x64;c:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\lib\amd64;C:\Program Files (x86)\Microsoft Visual Studio 10.0\;C:\Program Files (x86)\Microsoft Visual Studio 10.0\lib\amd64;" + ) + + build_script: + - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%HOST%]==[mingw] ( + ( if [%PLATFORM%]==[x64] ( + SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%" + ) else if [%PLATFORM%]==[x86] ( + SET "PATH=%PATH_MINGW32%;%PATH_ORIGINAL%" + ) ) && + make -v && + sh -c "%COMPILER% -v" && + set "CC=%COMPILER%" && + sh -c "%SCRIPT%" + ) + - if [%HOST%]==[visual] ( + ECHO *** && + ECHO *** Building Visual Studio 2015 %PLATFORM%\%CONFIGURATION% && + ECHO *** && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /p:ForceImportBeforeCppTargets=%APPVEYOR_BUILD_FOLDER%\build\VS2010\CompileAsCpp.props /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && + msbuild "build\VS2010\zstd.sln" /m /verbosity:minimal /property:PlatformToolset=v140 /t:Clean,Build /p:Platform=%PLATFORM% /p:Configuration=%CONFIGURATION% /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" && + DIR build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe && + MD5sum build/VS2010/bin/%PLATFORM%_%CONFIGURATION%/*.exe && + COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\fuzzer.exe tests\fuzzer_VS2015_%PLATFORM%_%CONFIGURATION%.exe && + COPY build\VS2010\bin\%PLATFORM%_%CONFIGURATION%\*.exe tests\ + ) Modified: head/contrib/zstd/circle.yml ============================================================================== --- head/contrib/zstd/circle.yml Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/circle.yml Sat May 6 10:26:40 2017 (r317877) @@ -3,7 +3,7 @@ dependencies: - sudo dpkg --add-architecture i386 - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test; sudo apt-get -y -qq update - sudo apt-get -y install gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross - - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 + - sudo apt-get -y install libstdc++-6-dev clang gcc g++ gcc-5 gcc-6 zlib1g-dev liblzma-dev - sudo apt-get -y install linux-libc-dev:i386 libc6-dev-i386 test: Copied: head/contrib/zstd/contrib/cleanTabs (from r317876, vendor/zstd/dist/contrib/cleanTabs) ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ head/contrib/zstd/contrib/cleanTabs Sat May 6 10:26:40 2017 (r317877, copy of r317876, vendor/zstd/dist/contrib/cleanTabs) @@ -0,0 +1,2 @@ +#!/bin/sh +sed -i '' $'s/\t/ /g' ../lib/**/*.{h,c} ../programs/*.{h,c} ../tests/*.c ./**/*.{h,cpp} ../examples/*.c ../zlibWrapper/*.{h,c} Modified: head/contrib/zstd/contrib/pzstd/Options.cpp ============================================================================== --- head/contrib/zstd/contrib/pzstd/Options.cpp Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/contrib/pzstd/Options.cpp Sat May 6 10:26:40 2017 (r317877) @@ -91,7 +91,7 @@ void usage() { std::fprintf(stderr, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel); std::fprintf(stderr, " -d, --decompress : decompression\n"); std::fprintf(stderr, " -o file : result stored into `file` (only if 1 input file)\n"); - std::fprintf(stderr, " -f, --force : overwrite output without prompting\n"); + std::fprintf(stderr, " -f, --force : overwrite output without prompting, (de)compress links\n"); std::fprintf(stderr, " --rm : remove source file(s) after successful (de)compression\n"); std::fprintf(stderr, " -k, --keep : preserve source file(s) (default)\n"); std::fprintf(stderr, " -h, --help : display help and exit\n"); @@ -121,6 +121,7 @@ Options::Status Options::parse(int argc, bool recursive = false; bool ultra = false; bool forceStdout = false; + bool followLinks = false; // Local copy of input files, which are pointers into argv. std::vector<const char *> localInputFiles; for (int i = 1; i < argc; ++i) { @@ -255,6 +256,7 @@ Options::Status Options::parse(int argc, case 'f': overwrite = true; forceStdout = true; + followLinks = true; break; case 't': test = true; @@ -328,13 +330,29 @@ Options::Status Options::parse(int argc, } } + g_utilDisplayLevel = verbosity; + // Remove local input files that are symbolic links + if (!followLinks) { + std::remove_if(localInputFiles.begin(), localInputFiles.end(), + [&](const char *path) { + bool isLink = UTIL_isLink(path); + if (isLink && verbosity >= 2) { + std::fprintf( + stderr, + "Warning : %s is symbolic link, ignoring\n", + path); + } + return isLink; + }); + } + // Translate input files/directories into files to (de)compress if (recursive) { char *scratchBuffer = nullptr; unsigned numFiles = 0; const char **files = UTIL_createFileList(localInputFiles.data(), localInputFiles.size(), - &scratchBuffer, &numFiles); + &scratchBuffer, &numFiles, followLinks); if (files == nullptr) { std::fprintf(stderr, "Error traversing directories\n"); return Status::Failure; Modified: head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp ============================================================================== --- head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp Sat May 6 10:26:40 2017 (r317877) @@ -10,6 +10,7 @@ #include <gtest/gtest.h> #include <atomic> +#include <iostream> #include <thread> #include <vector> @@ -34,16 +35,19 @@ TEST(ThreadPool, AllJobsFinished) { std::atomic<unsigned> numFinished{0}; std::atomic<bool> start{false}; { + std::cerr << "Creating executor" << std::endl; ThreadPool executor(5); for (int i = 0; i < 10; ++i) { executor.add([ &numFinished, &start ] { while (!start.load()) { - // spin + std::this_thread::yield(); } ++numFinished; }); } + std::cerr << "Starting" << std::endl; start.store(true); + std::cerr << "Finishing" << std::endl; } EXPECT_EQ(10, numFinished.load()); } Modified: head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp ============================================================================== --- head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp Sat May 6 10:26:40 2017 (r317877) @@ -10,6 +10,7 @@ #include "utils/WorkQueue.h" #include <gtest/gtest.h> +#include <iostream> #include <memory> #include <mutex> #include <thread> @@ -201,11 +202,13 @@ TEST(WorkQueue, BoundedSizeMPMC) { WorkQueue<int> queue(10); std::vector<int> results(200, -1); std::mutex mutex; + std::cerr << "Creating popperThreads" << std::endl; std::vector<std::thread> popperThreads; for (int i = 0; i < 4; ++i) { popperThreads.emplace_back(Popper{&queue, results.data(), &mutex}); } + std::cerr << "Creating pusherThreads" << std::endl; std::vector<std::thread> pusherThreads; for (int i = 0; i < 2; ++i) { auto min = i * 100; @@ -218,15 +221,19 @@ TEST(WorkQueue, BoundedSizeMPMC) { }); } + std::cerr << "Joining pusherThreads" << std::endl; for (auto& thread : pusherThreads) { thread.join(); } + std::cerr << "Finishing queue" << std::endl; queue.finish(); + std::cerr << "Joining popperThreads" << std::endl; for (auto& thread : popperThreads) { thread.join(); } + std::cerr << "Inspecting results" << std::endl; for (int i = 0; i < 200; ++i) { EXPECT_EQ(i, results[i]); } Modified: head/contrib/zstd/doc/educational_decoder/zstd_decompress.c ============================================================================== --- head/contrib/zstd/doc/educational_decoder/zstd_decompress.c Sat May 6 10:18:45 2017 (r317876) +++ head/contrib/zstd/doc/educational_decoder/zstd_decompress.c Sat May 6 10:26:40 2017 (r317877) @@ -27,16 +27,19 @@ size_t ZSTD_decompress_with_dict(void *c /// Get the decompressed size of an input stream so memory can be allocated in /// advance +/// Returns -1 if the size can't be determined size_t ZSTD_get_decompressed_size(const void *const src, const size_t src_len); /******* UTILITY MACROS AND TYPES *********************************************/ -// Max block size decompressed size is 128 KB and literal blocks must be smaller -// than that +// Max block size decompressed size is 128 KB and literal blocks can't be +// larger than their block #define MAX_LITERALS_SIZE ((size_t)128 * 1024) #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b)) +/// This decoder calls exit(1) when it encounters an error, however a production +/// library should propagate error codes #define ERROR(s) \ do { \ fprintf(stderr, "Error: %s\n", s); \ @@ -67,29 +70,31 @@ typedef int64_t i64; /// decompression functions. /*** IO STREAM OPERATIONS *************/ -/// These structs are the interface for IO, and do bounds checking on all -/// operations. They should be used opaquely to ensure safety. -/// Output is always done byte-by-byte +/// ostream_t/istream_t are used to wrap the pointers/length data passed into +/// ZSTD_decompress, so that all IO operations are safely bounds checked +/// They are written/read forward, and reads are treated as little-endian +/// They should be used opaquely to ensure safety typedef struct { u8 *ptr; size_t len; } ostream_t; -/// Input often reads a few bits at a time, so maintain an internal offset typedef struct { const u8 *ptr; - int bit_offset; size_t len; + + // Input often reads a few bits at a time, so maintain an internal offset + int bit_offset; } istream_t; /// The following two functions are the only ones that allow the istream to be /// non-byte aligned /// Reads `num` bits from a bitstream, and updates the internal offset -static inline u64 IO_read_bits(istream_t *const in, const int num); -/// Rewinds the stream by `num` bits -static inline void IO_rewind_bits(istream_t *const in, const int num); +static inline u64 IO_read_bits(istream_t *const in, const int num_bits); +/// Backs-up the stream by `num` bits so they can be read again +static inline void IO_rewind_bits(istream_t *const in, const int num_bits); /// If the remaining bits in a byte will be unused, advance to the end of the /// byte static inline void IO_align_stream(istream_t *const in); @@ -101,30 +106,31 @@ static inline void IO_write_byte(ostream /// be byte aligned. static inline size_t IO_istream_len(const istream_t *const in); -/// Returns a pointer where `len` bytes can be read, and advances the internal -/// state. The stream must be byte aligned. +/// Advances the stream by `len` bytes, and returns a pointer to the chunk that +/// was skipped. The stream must be byte aligned. static inline const u8 *IO_read_bytes(istream_t *const in, size_t len); -/// Returns a pointer where `len` bytes can be written, and advances the internal -/// state. The stream must be byte aligned. +/// Advances the stream by `len` bytes, and returns a pointer to the chunk that +/// was skipped so it can be written to. static inline u8 *IO_write_bytes(ostream_t *const out, size_t len); /// Advance the inner state by `len` bytes. The stream must be byte aligned. static inline void IO_advance_input(istream_t *const in, size_t len); -/// Returns an `ostream_t` constructed from the given pointer and length +/// Returns an `ostream_t` constructed from the given pointer and length. static inline ostream_t IO_make_ostream(u8 *out, size_t len); -/// Returns an `istream_t` constructed from the given pointer and length +/// Returns an `istream_t` constructed from the given pointer and length. static inline istream_t IO_make_istream(const u8 *in, size_t len); -/// Returns an `istream_t` with the same base as `in`, and length `len` -/// Then, advance `in` to account for the consumed bytes -/// `in` must be byte aligned +/// Returns an `istream_t` with the same base as `in`, and length `len`. +/// Then, advance `in` to account for the consumed bytes. +/// `in` must be byte aligned. static inline istream_t IO_make_sub_istream(istream_t *const in, size_t len); /*** END IO STREAM OPERATIONS *********/ /*** BITSTREAM OPERATIONS *************/ -/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits -static inline u64 read_bits_LE(const u8 *src, const int num, +/// Read `num` bits (up to 64) from `src + offset`, where `offset` is in bits, +/// and return them interpreted as a little-endian unsigned integer. +static inline u64 read_bits_LE(const u8 *src, const int num_bits, const size_t offset); /// Read bits from the end of a HUF or FSE bitstream. `offset` is in bits, so @@ -136,9 +142,8 @@ static inline u64 STREAM_read_bits(const /*** END BITSTREAM OPERATIONS *********/ /*** BIT COUNTING OPERATIONS **********/ -/// Returns `x`, where `2^x` is the largest power of 2 less than or equal to -/// `num`, or `-1` if `num == 0`. -static inline int log2inf(const u64 num); +/// Returns the index of the highest set bit in `num`, or `-1` if `num == 0` +static inline int highest_set_bit(const u64 num); /*** END BIT COUNTING OPERATIONS ******/ /*** HUFFMAN PRIMITIVES ***************/ @@ -384,8 +389,8 @@ size_t ZSTD_decompress_with_dict(void *c parse_dictionary(&parsed_dict, (const u8 *)dict, dict_len); } - istream_t in = {(const u8 *)src, 0, src_len}; - ostream_t out = {(u8 *)dst, dst_len}; + istream_t in = IO_make_istream(src, src_len); + ostream_t out = IO_make_ostream(dst, dst_len); // "A content compressed by Zstandard is transformed into a Zstandard frame. // Multiple frames can be appended into a single file or stream. A frame is @@ -633,6 +638,7 @@ static void frame_context_apply_dict(fra FSE_copy_dtable(&ctx->of_dtable, &dict->of_dtable); FSE_copy_dtable(&ctx->ml_dtable, &dict->ml_dtable); + // Copy the repeated offsets memcpy(ctx->previous_offsets, dict->previous_offsets, sizeof(ctx->previous_offsets)); } @@ -668,7 +674,7 @@ static void decompress_data(frame_contex // number of bytes to read and copy." const u8 *const read_ptr = IO_read_bytes(in, block_len); u8 *const write_ptr = IO_write_bytes(out, block_len); - // + // Copy the raw data into the output memcpy(write_ptr, read_ptr, block_len); @@ -682,7 +688,7 @@ static void decompress_data(frame_contex const u8 *const read_ptr = IO_read_bytes(in, 1); u8 *const write_ptr = IO_write_bytes(out, block_len); - // Copy `block_len` copies of `streams->src[0]` to the output + // Copy `block_len` copies of `read_ptr[0]` to the output memset(write_ptr, read_ptr[0], block_len); ctx->current_total_output += block_len; @@ -751,7 +757,7 @@ static size_t decode_literals_compressed u8 **const literals, const int block_type, const int size_format); -static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable); +static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in); static void fse_decode_hufweights(ostream_t *weights, istream_t *const in, int *const num_symbs); @@ -894,12 +900,12 @@ static size_t decode_literals_compressed istream_t huf_stream = IO_make_sub_istream(in, compressed_size); if (block_type == 2) { - // Decode provided Huffman table + // Decode the provided Huffman table // "This section is only present when Literals_Block_Type type is // Compressed_Literals_Block (2)." HUF_free_dtable(&ctx->literals_dtable); - decode_huf_table(&huf_stream, &ctx->literals_dtable); + decode_huf_table(&ctx->literals_dtable, &huf_stream); } else { // If the previous Huffman table is being repeated, ensure it exists if (!ctx->literals_dtable.symbols) { @@ -922,13 +928,13 @@ static size_t decode_literals_compressed } // Decode the Huffman table description -static void decode_huf_table(istream_t *const in, HUF_dtable *const dtable) { - const u8 header = IO_read_bits(in, 8); - +static void decode_huf_table(HUF_dtable *const dtable, istream_t *const in) { // "All literal values from zero (included) to last present one (excluded) // are represented by Weight with values from 0 to Max_Number_of_Bits." // "This is a single byte value (0-255), which describes how to decode the list of weights." + const u8 header = IO_read_bits(in, 8); + u8 weights[HUF_MAX_SYMBS]; memset(weights, 0, sizeof(weights)); @@ -997,7 +1003,7 @@ typedef struct { u16 ll_state; u16 of_state; u16 ml_state; -} sequence_state_t; +} sequence_states_t; /// Different modes to signal to decode_seq_tables what to do typedef enum { @@ -1052,10 +1058,10 @@ static void decompress_sequences(frame_c istream_t *const in, sequence_command_t *const sequences, const size_t num_sequences); -static sequence_command_t decode_sequence(sequence_state_t *const state, +static sequence_command_t decode_sequence(sequence_states_t *const state, const u8 *const src, i64 *const offset); -static void decode_seq_table(istream_t *const in, FSE_dtable *const table, +static void decode_seq_table(FSE_dtable *const table, istream_t *const in, const seq_part_t type, const seq_mode_t mode); static size_t decode_sequences(frame_context_t *const ctx, istream_t *in, @@ -1131,34 +1137,33 @@ static void decompress_sequences(frame_c // Offsets // Match Lengths" // Update the tables we have stored in the context - decode_seq_table(in, &ctx->ll_dtable, seq_literal_length, + decode_seq_table(&ctx->ll_dtable, in, seq_literal_length, (compression_modes >> 6) & 3); - decode_seq_table(in, &ctx->of_dtable, seq_offset, + decode_seq_table(&ctx->of_dtable, in, seq_offset, (compression_modes >> 4) & 3); - decode_seq_table(in, &ctx->ml_dtable, seq_match_length, + decode_seq_table(&ctx->ml_dtable, in, seq_match_length, (compression_modes >> 2) & 3); - // Check to make sure none of the tables are uninitialized - if (!ctx->ll_dtable.symbols || !ctx->of_dtable.symbols || - !ctx->ml_dtable.symbols) { - CORRUPTION(); - } - sequence_state_t state; - // Copy the context's tables into the local state - memcpy(&state.ll_table, &ctx->ll_dtable, sizeof(FSE_dtable)); - memcpy(&state.of_table, &ctx->of_dtable, sizeof(FSE_dtable)); - memcpy(&state.ml_table, &ctx->ml_dtable, sizeof(FSE_dtable)); + sequence_states_t states; + + // Initialize the decoding tables + { + states.ll_table = ctx->ll_dtable; + states.of_table = ctx->of_dtable; + states.ml_table = ctx->ml_dtable; + } - size_t len = IO_istream_len(in); + const size_t len = IO_istream_len(in); const u8 *const src = IO_read_bytes(in, len); // "After writing the last bit containing information, the compressor writes // a single 1-bit and then fills the byte with 0-7 0 bits of padding." - const int padding = 8 - log2inf(src[len - 1]); - i64 offset = len * 8 - padding; + const int padding = 8 - highest_set_bit(src[len - 1]); + // The offset starts at the end because FSE streams are read backwards + i64 bit_offset = len * 8 - padding; // "The bitstream starts with initial state values, each using the required // number of bits in their respective accuracy, decoded previously from @@ -1166,24 +1171,22 @@ static void decompress_sequences(frame_c // // It starts by Literals_Length_State, followed by Offset_State, and finally // Match_Length_State." - FSE_init_state(&state.ll_table, &state.ll_state, src, &offset); - FSE_init_state(&state.of_table, &state.of_state, src, &offset); - FSE_init_state(&state.ml_table, &state.ml_state, src, &offset); + FSE_init_state(&states.ll_table, &states.ll_state, src, &bit_offset); + FSE_init_state(&states.of_table, &states.of_state, src, &bit_offset); + FSE_init_state(&states.ml_table, &states.ml_state, src, &bit_offset); for (size_t i = 0; i < num_sequences; i++) { // Decode sequences one by one - sequences[i] = decode_sequence(&state, src, &offset); + sequences[i] = decode_sequence(&states, src, &bit_offset); } - if (offset != 0) { + if (bit_offset != 0) { CORRUPTION(); } - - // Don't free tables so they can be used in the next block } // Decode a single sequence and update the state -static sequence_command_t decode_sequence(sequence_state_t *const state, +static sequence_command_t decode_sequence(sequence_states_t *const states, const u8 *const src, i64 *const offset) { // "Each symbol is a code in its own context, which specifies Baseline and @@ -1191,9 +1194,9 @@ static sequence_command_t decode_sequenc // additional bits in the same bitstream." // Decode symbols, but don't update states - const u8 of_code = FSE_peek_symbol(&state->of_table, state->of_state); - const u8 ll_code = FSE_peek_symbol(&state->ll_table, state->ll_state); - const u8 ml_code = FSE_peek_symbol(&state->ml_table, state->ml_state); + const u8 of_code = FSE_peek_symbol(&states->of_table, states->of_state); + const u8 ll_code = FSE_peek_symbol(&states->ll_table, states->ll_state); + const u8 ml_code = FSE_peek_symbol(&states->ml_table, states->ml_state); // Offset doesn't need a max value as it's not decoded using a table if (ll_code > SEQ_MAX_CODES[seq_literal_length] || @@ -1221,17 +1224,18 @@ static sequence_command_t decode_sequenc // then Offset_State." // If the stream is complete don't read bits to update state if (*offset != 0) { - FSE_update_state(&state->ll_table, &state->ll_state, src, offset); - FSE_update_state(&state->ml_table, &state->ml_state, src, offset); - FSE_update_state(&state->of_table, &state->of_state, src, offset); + FSE_update_state(&states->ll_table, &states->ll_state, src, offset); + FSE_update_state(&states->ml_table, &states->ml_state, src, offset); + FSE_update_state(&states->of_table, &states->of_state, src, offset); } return seq; } /// Given a sequence part and table mode, decode the FSE distribution -static void decode_seq_table(istream_t *const in, FSE_dtable *const table, - const seq_part_t type, const seq_mode_t mode) { +/// Errors if the mode is `seq_repeat` without a pre-existing table in `table` +static void decode_seq_table(FSE_dtable *const table, istream_t *const in, + const seq_part_t type, const seq_mode_t mode) { // Constant arrays indexed by seq_part_t const i16 *const default_distributions[] = {SEQ_LITERAL_LENGTH_DEFAULT_DIST, SEQ_OFFSET_DEFAULT_DIST, @@ -1272,12 +1276,17 @@ static void decode_seq_table(istream_t * // "Repeat_Mode : re-use distribution table from previous compressed // block." // Nothing to do here, table will be unchanged + if (!table->symbols) { + // This mode is invalid if we don't already have a table + CORRUPTION(); + } break; default: // Impossible, as mode is from 0-3 IMPOSSIBLE(); break; } + } /******* END SEQUENCE DECODING ************************************************/ @@ -1296,6 +1305,8 @@ static void execute_sequences(frame_cont const sequence_command_t seq = sequences[i]; { + // If the sequence asks for more literals than are left, the + // sequence must be corrupted if (seq.literal_length > IO_istream_len(&litstream)) { CORRUPTION(); } @@ -1336,7 +1347,8 @@ static void execute_sequences(frame_cont // as per the exception listed above offset = idx < 3 ? offset_hist[idx] : offset_hist[0] - 1; *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
](https://travis-ci.org/facebook/zstd){kind=link}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201705061026.v46AQesA089962>