Date: Sat, 23 May 2020 21:23:46 +0000 (UTC) From: Conrad Meyer <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r361426 - in head: lib/libzstd sys/conf sys/contrib/zstd sys/contrib/zstd/contrib sys/contrib/zstd/contrib/docker sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilde... Message-ID: <202005232123.04NLNkG4078947@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Sat May 23 21:23:46 2020 New Revision: 361426 URL: https://svnweb.freebsd.org/changeset/base/361426 Log: Update to Zstandard 1.4.5 As usual, the full release notes are found on Github: https://github.com/facebook/zstd/releases/tag/v1.4.5 Notable changes include: * Improved decompress performance on amd64 and arm (5-10% and 15-50%, respectively). * '--patch-from' zstd(1) CLI option, which provides something like a very fast version of bspatch(1) with slightly worse compression. See release notes. In this update, I dropped the 3-year old -O0 workaround for an LLVM ARM bug; the bug was fixed in LLVM SVN in 2017, but we didn't remove this workaround from our tree until now. MFC after: I won't, but feel free Relnotes: yes Added: head/sys/contrib/zstd/lib/compress/zstd_compress_superblock.c - copied unchanged from r361425, vendor/zstd/dist/lib/compress/zstd_compress_superblock.c head/sys/contrib/zstd/lib/compress/zstd_compress_superblock.h - copied unchanged from r361425, vendor/zstd/dist/lib/compress/zstd_compress_superblock.h Deleted: head/sys/contrib/zstd/contrib/cleanTabs head/sys/contrib/zstd/contrib/docker/Dockerfile head/sys/contrib/zstd/contrib/docker/README.md head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/Makefile head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/README.md head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/benchmark.c head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/dictBuilder.h head/sys/contrib/zstd/contrib/experimental_dict_builders/benchmarkDictBuilder/test.sh head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/Makefile head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/README.md head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/fastCover.c head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/fastCover.h head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/main.c head/sys/contrib/zstd/contrib/experimental_dict_builders/fastCover/test.sh head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/Makefile head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/README.md head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/io.c head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/io.h head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/main.c head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/random.c head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/random.h head/sys/contrib/zstd/contrib/experimental_dict_builders/randomDictBuilder/test.sh head/sys/contrib/zstd/contrib/gen_html/Makefile head/sys/contrib/zstd/contrib/gen_html/README.md head/sys/contrib/zstd/contrib/gen_html/gen-zstd-manual.sh head/sys/contrib/zstd/contrib/gen_html/gen_html.cpp head/sys/contrib/zstd/contrib/largeNbDicts/Makefile head/sys/contrib/zstd/contrib/largeNbDicts/README.md head/sys/contrib/zstd/contrib/largeNbDicts/largeNbDicts.c head/sys/contrib/zstd/contrib/premake/premake4.lua head/sys/contrib/zstd/contrib/premake/zstd.lua head/sys/contrib/zstd/contrib/pzstd/BUCK head/sys/contrib/zstd/contrib/pzstd/ErrorHolder.h head/sys/contrib/zstd/contrib/pzstd/Logging.h head/sys/contrib/zstd/contrib/pzstd/Makefile head/sys/contrib/zstd/contrib/pzstd/Options.cpp head/sys/contrib/zstd/contrib/pzstd/Options.h head/sys/contrib/zstd/contrib/pzstd/Pzstd.cpp head/sys/contrib/zstd/contrib/pzstd/Pzstd.h head/sys/contrib/zstd/contrib/pzstd/README.md head/sys/contrib/zstd/contrib/pzstd/SkippableFrame.cpp head/sys/contrib/zstd/contrib/pzstd/SkippableFrame.h head/sys/contrib/zstd/contrib/pzstd/images/Cspeed.png head/sys/contrib/zstd/contrib/pzstd/images/Dspeed.png head/sys/contrib/zstd/contrib/pzstd/main.cpp head/sys/contrib/zstd/contrib/pzstd/test/BUCK head/sys/contrib/zstd/contrib/pzstd/test/OptionsTest.cpp head/sys/contrib/zstd/contrib/pzstd/test/PzstdTest.cpp head/sys/contrib/zstd/contrib/pzstd/test/RoundTrip.h head/sys/contrib/zstd/contrib/pzstd/test/RoundTripTest.cpp head/sys/contrib/zstd/contrib/pzstd/utils/BUCK head/sys/contrib/zstd/contrib/pzstd/utils/Buffer.h head/sys/contrib/zstd/contrib/pzstd/utils/FileSystem.h head/sys/contrib/zstd/contrib/pzstd/utils/Likely.h head/sys/contrib/zstd/contrib/pzstd/utils/Range.h head/sys/contrib/zstd/contrib/pzstd/utils/ResourcePool.h head/sys/contrib/zstd/contrib/pzstd/utils/ScopeGuard.h head/sys/contrib/zstd/contrib/pzstd/utils/ThreadPool.h head/sys/contrib/zstd/contrib/pzstd/utils/WorkQueue.h head/sys/contrib/zstd/contrib/pzstd/utils/test/BUCK head/sys/contrib/zstd/contrib/pzstd/utils/test/BufferTest.cpp head/sys/contrib/zstd/contrib/pzstd/utils/test/RangeTest.cpp head/sys/contrib/zstd/contrib/pzstd/utils/test/ResourcePoolTest.cpp head/sys/contrib/zstd/contrib/pzstd/utils/test/ScopeGuardTest.cpp head/sys/contrib/zstd/contrib/pzstd/utils/test/ThreadPoolTest.cpp head/sys/contrib/zstd/contrib/pzstd/utils/test/WorkQueueTest.cpp head/sys/contrib/zstd/contrib/seekable_format/examples/Makefile head/sys/contrib/zstd/contrib/seekable_format/examples/parallel_compression.c head/sys/contrib/zstd/contrib/seekable_format/examples/parallel_processing.c head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_compression.c head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression.c head/sys/contrib/zstd/contrib/seekable_format/examples/seekable_decompression_mem.c head/sys/contrib/zstd/contrib/seekable_format/zstd_seekable.h head/sys/contrib/zstd/contrib/seekable_format/zstd_seekable_compression_format.md head/sys/contrib/zstd/contrib/seekable_format/zstdseek_compress.c head/sys/contrib/zstd/contrib/seekable_format/zstdseek_decompress.c head/sys/contrib/zstd/contrib/snap/snapcraft.yaml head/sys/contrib/zstd/tests/Makefile head/sys/contrib/zstd/tests/README.md head/sys/contrib/zstd/tests/bigdict.c head/sys/contrib/zstd/tests/checkTag.c head/sys/contrib/zstd/tests/datagencli.c head/sys/contrib/zstd/tests/decodecorpus.c head/sys/contrib/zstd/tests/fullbench.c head/sys/contrib/zstd/tests/fuzz/Makefile head/sys/contrib/zstd/tests/fuzz/README.md head/sys/contrib/zstd/tests/fuzz/block_decompress.c head/sys/contrib/zstd/tests/fuzz/block_round_trip.c head/sys/contrib/zstd/tests/fuzz/dictionary_decompress.c head/sys/contrib/zstd/tests/fuzz/dictionary_loader.c head/sys/contrib/zstd/tests/fuzz/dictionary_round_trip.c head/sys/contrib/zstd/tests/fuzz/fuzz.h head/sys/contrib/zstd/tests/fuzz/fuzz.py head/sys/contrib/zstd/tests/fuzz/fuzz_data_producer.c head/sys/contrib/zstd/tests/fuzz/fuzz_data_producer.h head/sys/contrib/zstd/tests/fuzz/fuzz_helpers.h head/sys/contrib/zstd/tests/fuzz/regression_driver.c head/sys/contrib/zstd/tests/fuzz/simple_compress.c head/sys/contrib/zstd/tests/fuzz/simple_decompress.c head/sys/contrib/zstd/tests/fuzz/simple_round_trip.c head/sys/contrib/zstd/tests/fuzz/stream_decompress.c head/sys/contrib/zstd/tests/fuzz/stream_round_trip.c head/sys/contrib/zstd/tests/fuzz/zstd_frame_info.c head/sys/contrib/zstd/tests/fuzz/zstd_helpers.c head/sys/contrib/zstd/tests/fuzz/zstd_helpers.h head/sys/contrib/zstd/tests/fuzzer.c head/sys/contrib/zstd/tests/golden-compression/huffman-compressed-larger head/sys/contrib/zstd/tests/golden-decompression/rle-first-block.zst head/sys/contrib/zstd/tests/gzip/Makefile head/sys/contrib/zstd/tests/gzip/gzip-env.sh head/sys/contrib/zstd/tests/gzip/helin-segv.sh head/sys/contrib/zstd/tests/gzip/help-version.sh head/sys/contrib/zstd/tests/gzip/hufts-segv.gz head/sys/contrib/zstd/tests/gzip/hufts.sh head/sys/contrib/zstd/tests/gzip/init.cfg head/sys/contrib/zstd/tests/gzip/init.sh head/sys/contrib/zstd/tests/gzip/keep.sh head/sys/contrib/zstd/tests/gzip/list.sh head/sys/contrib/zstd/tests/gzip/memcpy-abuse.sh head/sys/contrib/zstd/tests/gzip/mixed.sh head/sys/contrib/zstd/tests/gzip/null-suffix-clobber.sh head/sys/contrib/zstd/tests/gzip/stdin.sh head/sys/contrib/zstd/tests/gzip/test-driver.sh head/sys/contrib/zstd/tests/gzip/trailing-nul.sh head/sys/contrib/zstd/tests/gzip/unpack-invalid.sh head/sys/contrib/zstd/tests/gzip/z-suffix.sh head/sys/contrib/zstd/tests/gzip/zdiff.sh head/sys/contrib/zstd/tests/gzip/zgrep-context.sh head/sys/contrib/zstd/tests/gzip/zgrep-f.sh head/sys/contrib/zstd/tests/gzip/zgrep-signal.sh head/sys/contrib/zstd/tests/gzip/znew-k.sh head/sys/contrib/zstd/tests/invalidDictionaries.c head/sys/contrib/zstd/tests/legacy.c head/sys/contrib/zstd/tests/libzstd_partial_builds.sh head/sys/contrib/zstd/tests/longmatch.c head/sys/contrib/zstd/tests/paramgrill.c head/sys/contrib/zstd/tests/playTests.sh head/sys/contrib/zstd/tests/poolTests.c head/sys/contrib/zstd/tests/rateLimiter.py head/sys/contrib/zstd/tests/regression/Makefile head/sys/contrib/zstd/tests/regression/config.c head/sys/contrib/zstd/tests/regression/config.h head/sys/contrib/zstd/tests/regression/data.c head/sys/contrib/zstd/tests/regression/data.h head/sys/contrib/zstd/tests/regression/levels.h head/sys/contrib/zstd/tests/regression/method.c head/sys/contrib/zstd/tests/regression/method.h head/sys/contrib/zstd/tests/regression/result.c head/sys/contrib/zstd/tests/regression/result.h head/sys/contrib/zstd/tests/regression/results.csv head/sys/contrib/zstd/tests/regression/test.c head/sys/contrib/zstd/tests/roundTripCrash.c head/sys/contrib/zstd/tests/seqgen.c head/sys/contrib/zstd/tests/seqgen.h head/sys/contrib/zstd/tests/symbols.c head/sys/contrib/zstd/tests/test-zstd-speed.py head/sys/contrib/zstd/tests/test-zstd-versions.py head/sys/contrib/zstd/tests/zbufftest.c head/sys/contrib/zstd/tests/zstreamtest.c Modified: head/lib/libzstd/Makefile head/sys/conf/files head/sys/contrib/zstd/CHANGELOG head/sys/contrib/zstd/CONTRIBUTING.md head/sys/contrib/zstd/Makefile head/sys/contrib/zstd/README.md head/sys/contrib/zstd/TESTING.md head/sys/contrib/zstd/appveyor.yml head/sys/contrib/zstd/doc/educational_decoder/Makefile head/sys/contrib/zstd/doc/educational_decoder/README.md head/sys/contrib/zstd/doc/educational_decoder/harness.c head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.h head/sys/contrib/zstd/doc/zstd_compression_format.md head/sys/contrib/zstd/doc/zstd_manual.html head/sys/contrib/zstd/examples/Makefile head/sys/contrib/zstd/examples/common.h head/sys/contrib/zstd/examples/dictionary_compression.c head/sys/contrib/zstd/examples/dictionary_decompression.c head/sys/contrib/zstd/examples/multiple_simple_compression.c head/sys/contrib/zstd/examples/multiple_streaming_compression.c head/sys/contrib/zstd/examples/simple_compression.c head/sys/contrib/zstd/examples/simple_decompression.c head/sys/contrib/zstd/examples/streaming_compression.c head/sys/contrib/zstd/examples/streaming_decompression.c head/sys/contrib/zstd/examples/streaming_memory_usage.c head/sys/contrib/zstd/lib/Makefile head/sys/contrib/zstd/lib/README.md head/sys/contrib/zstd/lib/common/bitstream.h head/sys/contrib/zstd/lib/common/compiler.h head/sys/contrib/zstd/lib/common/cpu.h head/sys/contrib/zstd/lib/common/debug.c head/sys/contrib/zstd/lib/common/debug.h head/sys/contrib/zstd/lib/common/entropy_common.c head/sys/contrib/zstd/lib/common/error_private.c head/sys/contrib/zstd/lib/common/error_private.h head/sys/contrib/zstd/lib/common/fse.h head/sys/contrib/zstd/lib/common/fse_decompress.c head/sys/contrib/zstd/lib/common/huf.h head/sys/contrib/zstd/lib/common/mem.h head/sys/contrib/zstd/lib/common/pool.c head/sys/contrib/zstd/lib/common/pool.h head/sys/contrib/zstd/lib/common/threading.c head/sys/contrib/zstd/lib/common/threading.h head/sys/contrib/zstd/lib/common/xxhash.c head/sys/contrib/zstd/lib/common/xxhash.h head/sys/contrib/zstd/lib/common/zstd_common.c head/sys/contrib/zstd/lib/common/zstd_errors.h head/sys/contrib/zstd/lib/common/zstd_internal.h head/sys/contrib/zstd/lib/compress/fse_compress.c head/sys/contrib/zstd/lib/compress/hist.c head/sys/contrib/zstd/lib/compress/hist.h head/sys/contrib/zstd/lib/compress/huf_compress.c head/sys/contrib/zstd/lib/compress/zstd_compress.c head/sys/contrib/zstd/lib/compress/zstd_compress_internal.h head/sys/contrib/zstd/lib/compress/zstd_compress_literals.c head/sys/contrib/zstd/lib/compress/zstd_compress_literals.h head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.c head/sys/contrib/zstd/lib/compress/zstd_compress_sequences.h head/sys/contrib/zstd/lib/compress/zstd_cwksp.h head/sys/contrib/zstd/lib/compress/zstd_double_fast.c head/sys/contrib/zstd/lib/compress/zstd_double_fast.h head/sys/contrib/zstd/lib/compress/zstd_fast.c head/sys/contrib/zstd/lib/compress/zstd_fast.h head/sys/contrib/zstd/lib/compress/zstd_lazy.c head/sys/contrib/zstd/lib/compress/zstd_lazy.h head/sys/contrib/zstd/lib/compress/zstd_ldm.c head/sys/contrib/zstd/lib/compress/zstd_ldm.h head/sys/contrib/zstd/lib/compress/zstd_opt.c head/sys/contrib/zstd/lib/compress/zstd_opt.h head/sys/contrib/zstd/lib/compress/zstdmt_compress.c head/sys/contrib/zstd/lib/compress/zstdmt_compress.h head/sys/contrib/zstd/lib/decompress/huf_decompress.c head/sys/contrib/zstd/lib/decompress/zstd_ddict.c head/sys/contrib/zstd/lib/decompress/zstd_ddict.h head/sys/contrib/zstd/lib/decompress/zstd_decompress.c head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.c head/sys/contrib/zstd/lib/decompress/zstd_decompress_block.h head/sys/contrib/zstd/lib/decompress/zstd_decompress_internal.h head/sys/contrib/zstd/lib/deprecated/zbuff.h head/sys/contrib/zstd/lib/deprecated/zbuff_common.c head/sys/contrib/zstd/lib/deprecated/zbuff_compress.c head/sys/contrib/zstd/lib/deprecated/zbuff_decompress.c head/sys/contrib/zstd/lib/dictBuilder/cover.c head/sys/contrib/zstd/lib/dictBuilder/cover.h head/sys/contrib/zstd/lib/dictBuilder/fastcover.c head/sys/contrib/zstd/lib/dictBuilder/zdict.c head/sys/contrib/zstd/lib/dictBuilder/zdict.h head/sys/contrib/zstd/lib/legacy/zstd_legacy.h head/sys/contrib/zstd/lib/legacy/zstd_v01.c head/sys/contrib/zstd/lib/legacy/zstd_v01.h head/sys/contrib/zstd/lib/legacy/zstd_v02.c head/sys/contrib/zstd/lib/legacy/zstd_v02.h head/sys/contrib/zstd/lib/legacy/zstd_v03.c head/sys/contrib/zstd/lib/legacy/zstd_v03.h head/sys/contrib/zstd/lib/legacy/zstd_v04.c head/sys/contrib/zstd/lib/legacy/zstd_v04.h head/sys/contrib/zstd/lib/legacy/zstd_v05.c head/sys/contrib/zstd/lib/legacy/zstd_v05.h head/sys/contrib/zstd/lib/legacy/zstd_v06.c head/sys/contrib/zstd/lib/legacy/zstd_v06.h head/sys/contrib/zstd/lib/legacy/zstd_v07.c head/sys/contrib/zstd/lib/legacy/zstd_v07.h head/sys/contrib/zstd/lib/libzstd.pc.in head/sys/contrib/zstd/lib/zstd.h head/sys/contrib/zstd/programs/Makefile head/sys/contrib/zstd/programs/README.md head/sys/contrib/zstd/programs/benchfn.c head/sys/contrib/zstd/programs/benchfn.h head/sys/contrib/zstd/programs/benchzstd.c head/sys/contrib/zstd/programs/benchzstd.h head/sys/contrib/zstd/programs/datagen.c head/sys/contrib/zstd/programs/datagen.h head/sys/contrib/zstd/programs/dibio.c head/sys/contrib/zstd/programs/dibio.h head/sys/contrib/zstd/programs/fileio.c head/sys/contrib/zstd/programs/fileio.h head/sys/contrib/zstd/programs/platform.h head/sys/contrib/zstd/programs/timefn.c head/sys/contrib/zstd/programs/timefn.h head/sys/contrib/zstd/programs/util.c head/sys/contrib/zstd/programs/util.h head/sys/contrib/zstd/programs/zstd.1 head/sys/contrib/zstd/programs/zstd.1.md head/sys/contrib/zstd/programs/zstdcli.c head/sys/contrib/zstd/programs/zstdgrep head/sys/contrib/zstd/programs/zstdgrep.1 head/sys/contrib/zstd/programs/zstdless.1 head/sys/contrib/zstd/zlibWrapper/Makefile head/sys/contrib/zstd/zlibWrapper/examples/zwrapbench.c head/sys/contrib/zstd/zlibWrapper/gzcompatibility.h head/sys/contrib/zstd/zlibWrapper/zstd_zlibwrapper.c head/sys/contrib/zstd/zlibWrapper/zstd_zlibwrapper.h Directory Properties: head/sys/contrib/zstd/ (props changed) Modified: head/lib/libzstd/Makefile ============================================================================== --- head/lib/libzstd/Makefile Sat May 23 20:39:36 2020 (r361425) +++ head/lib/libzstd/Makefile Sat May 23 21:23:46 2020 (r361426) @@ -13,6 +13,7 @@ SRCS= entropy_common.c \ zstd_compress.c \ zstd_compress_literals.c \ zstd_compress_sequences.c \ + zstd_compress_superblock.c \ zstdmt_compress.c \ huf_decompress.c \ zstd_ddict.c \ @@ -54,13 +55,6 @@ ZSTDDIR= ${SRCTOP}/sys/contrib/zstd # turning off vectorization, which is why there were both." .if ${COMPILER_TYPE} == "gcc" CFLAGS.zstd_decompress_block.c+= -fno-tree-vectorize -.endif - -# Work around for LLVM bug 35023, https://bugs.llvm.org/show_bug.cgi?id=35023 -# Fixed in https://reviews.llvm.org/rL349935 ; not sure if we have that version -# in our LLVM or not yet. -.if ${MACHINE_ARCH:Marm*} != "" -CFLAGS.zstd_compress.c+= -O0 .endif # Work around lack of __bswap[sd]i2() intrinsics on riscv. Probably should be Modified: head/sys/conf/files ============================================================================== --- head/sys/conf/files Sat May 23 20:39:36 2020 (r361425) +++ head/sys/conf/files Sat May 23 21:23:46 2020 (r361426) @@ -659,6 +659,7 @@ contrib/zstd/lib/common/xxhash.c optional zstdio comp contrib/zstd/lib/compress/zstd_compress.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/zstd_compress_literals.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/zstd_compress_sequences.c optional zstdio compile-with ${ZSTD_C} +contrib/zstd/lib/compress/zstd_compress_superblock.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/fse_compress.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/hist.c optional zstdio compile-with ${ZSTD_C} contrib/zstd/lib/compress/huf_compress.c optional zstdio compile-with ${ZSTD_C} Modified: head/sys/contrib/zstd/CHANGELOG ============================================================================== --- head/sys/contrib/zstd/CHANGELOG Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/CHANGELOG Sat May 23 21:23:46 2020 (r361426) @@ -1,3 +1,29 @@ +v1.4.5 +fix : Compression ratio regression on huge files (> 3 GB) using high levels (--ultra) and multithreading, by @terrelln +perf: Improved decompression speed: x64 : +10% (clang) / +5% (gcc); ARM : from +15% to +50%, depending on SoC, by @terrelln +perf: Automatically downsizes ZSTD_DCtx when too large for too long (#2069, by @bimbashreshta) +perf: Improved fast compression speed on aarch64 (#2040, ~+3%, by @caoyzh) +perf: Small level 1 compression speed gains (depending on compiler) +cli : New --patch-from command, create and apply patches from files, by @bimbashreshta +cli : New --filelist= : Provide a list of files to operate upon from a file +cli : -b -d command can now benchmark decompression on multiple files +cli : New --no-content-size command +cli : New --show-default-cparams information command +api : ZDICT_finalizeDictionary() is promoted to stable (#2111) +api : new experimental parameter ZSTD_d_stableOutBuffer (#2094) +build: Generate a single-file libzstd library (#2065, by @cwoffenden) +build: Relative includes no longer require -I compiler flags for zstd lib subdirs (#2103, by @felixhandte) +build: zstd now compiles cleanly under -pedantic (#2099) +build: zstd now compiles with make-4.3 +build: Support mingw cross-compilation from Linux, by @Ericson2314 +build: Meson multi-thread build fix on windows +build: Some misc icc fixes backed by new ci test on travis +misc: bitflip analyzer tool, by @felixhandte +misc: Extend largeNbDicts benchmark to compression +misc: Edit-distance match finder in contrib/ +doc : Improved beginner CONTRIBUTING.md docs +doc : New issue templates for zstd + v1.4.4 perf: Improved decompression speed, by > 10%, by @terrelln perf: Better compression speed when re-using a context, by @felixhandte @@ -14,7 +40,8 @@ cli: commands --stream-size=# and --size-hint=#, by @n cli: command --exclude-compressed, by @shashank0791 cli: faster `-t` test mode cli: improved some error messages, by @vangyzen -cli: rare deadlock condition within dictionary builder, by @terrelln +cli: fix command `-D dictionary` on Windows, reported by @artyompetrov +cli: fix rare deadlock condition within dictionary builder, by @terrelln build: single-file decoder with emscripten compilation script, by @cwoffenden build: fixed zlibWrapper compilation on Visual Studio, reported by @bluenlive build: fixed deprecation warning for certain gcc version, reported by @jasonma163 Modified: head/sys/contrib/zstd/CONTRIBUTING.md ============================================================================== --- head/sys/contrib/zstd/CONTRIBUTING.md Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/CONTRIBUTING.md Sat May 23 21:23:46 2020 (r361426) @@ -26,6 +26,356 @@ to do this once to work on any of Facebook's open sour Complete your CLA here: <https://code.facebook.com/cla>; +## Workflow +Zstd uses a branch-based workflow for making changes to the codebase. Typically, zstd +will use a new branch per sizable topic. For smaller changes, it is okay to lump multiple +related changes into a branch. + +Our contribution process works in three main stages: +1. Local development + * Update: + * Checkout your fork of zstd if you have not already + ``` + git checkout https://github.com/<username>/zstd + cd zstd + ``` + * Update your local dev branch + ``` + git pull https://github.com/facebook/zstd dev + git push origin dev + ``` + * Topic and development: + * Make a new branch on your fork about the topic you're developing for + ``` + # branch names should be consise but sufficiently informative + git checkout -b <branch-name> + git push origin <branch-name> + ``` + * Make commits and push + ``` + # make some changes = + git add -u && git commit -m <message> + git push origin <branch-name> + ``` + * Note: run local tests to ensure that your changes didn't break existing functionality + * Quick check + ``` + make shortest + ``` + * Longer check + ``` + make test + ``` +2. Code Review and CI tests + * Ensure CI tests pass: + * Before sharing anything to the community, make sure that all CI tests pass on your local fork. + See our section on setting up your CI environment for more information on how to do this. + * Ensure that static analysis passes on your development machine. See the Static Analysis section + below to see how to do this. + * Create a pull request: + * When you are ready to share you changes to the community, create a pull request from your branch + to facebook:dev. You can do this very easily by clicking 'Create Pull Request' on your fork's home + page. + * From there, select the branch where you made changes as your source branch and facebook:dev + as the destination. + * Examine the diff presented between the two branches to make sure there is nothing unexpected. + * Write a good pull request description: + * While there is no strict template that our contributors follow, we would like them to + sufficiently summarize and motivate the changes they are proposing. We recommend all pull requests, + at least indirectly, address the following points. + * Is this pull request important and why? + * Is it addressing an issue? If so, what issue? (provide links for convenience please) + * Is this a new feature? If so, why is it useful and/or necessary? + * Are there background references and documents that reviewers should be aware of to properly assess this change? + * Note: make sure to point out any design and architectural decisions that you made and the rationale behind them. + * Note: if you have been working with a specific user and would like them to review your work, make sure you mention them using (@<username>) + * Submit the pull request and iterate with feedback. +3. Merge and Release + * Getting approval: + * You will have to iterate on your changes with feedback from other collaborators to reach a point + where your pull request can be safely merged. + * To avoid too many comments on style and convention, make sure that you have a + look at our style section below before creating a pull request. + * Eventually, someone from the zstd team will approve your pull request and not long after merge it into + the dev branch. + * Housekeeping: + * Most PRs are linked with one or more Github issues. If this is the case for your PR, make sure + the corresponding issue is mentioned. If your change 'fixes' or completely addresses the + issue at hand, then please indicate this by requesting that an issue be closed by commenting. + * Just because your changes have been merged does not mean the topic or larger issue is complete. Remember + that the change must make it to an official zstd release for it to be meaningful. We recommend + that contributers track the activity on their pull request and corresponding issue(s) page(s) until + their change makes it to the next release of zstd. Users will often discover bugs in your code or + suggest ways to refine and improve your initial changes even after the pull request is merged. + +## Static Analysis +Static analysis is a process for examining the correctness or validity of a program without actually +executing it. It usually helps us find many simple bugs. Zstd uses clang's `scan-build` tool for +static analysis. You can install it by following the instructions for your OS on https://clang-analyzer.llvm.org/scan-build. + +Once installed, you can ensure that our static analysis tests pass on your local development machine +by running: +``` +make staticAnalyze +``` + +In general, you can use `scan-build` to static analyze any build script. For example, to static analyze +just `contrib/largeNbDicts` and nothing else, you can run: + +``` +scan-build make -C contrib/largeNbDicts largeNbDicts +``` + +## Performance +Performance is extremely important for zstd and we only merge pull requests whose performance +landscape and corresponding trade-offs have been adequately analyzed, reproduced, and presented. +This high bar for performance means that every PR which has the potential to +impact performance takes a very long time for us to properly review. That being said, we +always welcome contributions to improve performance (or worsen performance for the trade-off of +something else). Please keep the following in mind before submitting a performance related PR: + +1. Zstd isn't as old as gzip but it has been around for time now and its evolution is +very well documented via past Github issues and pull requests. It may be the case that your +particular performance optimization has already been considered in the past. Please take some +time to search through old issues and pull requests using keywords specific to your +would-be PR. Of course, just because a topic has already been discussed (and perhaps rejected +on some grounds) in the past, doesn't mean it isn't worth bringing up again. But even in that case, +it will be helpful for you to have context from that topic's history before contributing. +2. The distinction between noise and actual performance gains can unfortunately be very subtle +especially when microbenchmarking extremely small wins or losses. The only remedy to getting +something subtle merged is extensive benchmarking. You will be doing us a great favor if you +take the time to run extensive, long-duration, and potentially cross-(os, platform, process, etc) +benchmarks on your end before submitting a PR. Of course, you will not be able to benchmark +your changes on every single processor and os out there (and neither will we) but do that best +you can:) We've adding some things to think about when benchmarking below in the Benchmarking +Performance section which might be helpful for you. +3. Optimizing performance for a certain OS, processor vendor, compiler, or network system is a perfectly +legitimate thing to do as long as it does not harm the overall performance health of Zstd. +This is a hard balance to strike but please keep in mind other aspects of Zstd when +submitting changes that are clang-specific, windows-specific, etc. + +## Benchmarking Performance +Performance microbenchmarking is a tricky subject but also essential for Zstd. We value empirical +testing over theoretical speculation. This guide it not perfect but for most scenarios, it +is a good place to start. + +### Stability +Unfortunately, the most important aspect in being able to benchmark reliably is to have a stable +benchmarking machine. A virtual machine, a machine with shared resources, or your laptop +will typically not be stable enough to obtain reliable benchmark results. If you can get your +hands on a desktop, this is usually a better scenario. + +Of course, benchmarking can be done on non-hyper-stable machines as well. You will just have to +do a little more work to ensure that you are in fact measuring the changes you've made not and +noise. Here are some things you can do to make your benchmarks more stable: + +1. The most simple thing you can do to drastically improve the stability of your benchmark is +to run it multiple times and then aggregate the results of those runs. As a general rule of +thumb, the smaller the change you are trying to measure, the more samples of benchmark runs +you will have to aggregate over to get reliable results. Here are some additional things to keep in +mind when running multiple trials: + * How you aggregate your samples are important. You might be tempted to use the mean of your + results. While this is certainly going to be a more stable number than a raw single sample + benchmark number, you might have more luck by taking the median. The mean is not robust to + outliers whereas the median is. Better still, you could simply take the fastest speed your + benchmark achieved on each run since that is likely the fastest your process will be + capable of running your code. In our experience, this (aggregating by just taking the sample + with the fastest running time) has been the most stable approach. + * The more samples you have, the more stable your benchmarks should be. You can verify + your improved stability by looking at the size of your confidence intervals as you + increase your sample count. These should get smaller and smaller. Eventually hopefully + smaller than the performance win you are expecting. + * Most processors will take some time to get `hot` when running anything. The observations + you collect during that time period will very different from the true performance number. Having + a very large number of sample will help alleviate this problem slightly but you can also + address is directly by simply not including the first `n` iterations of your benchmark in + your aggregations. You can determine `n` by simply looking at the results from each iteration + and then hand picking a good threshold after which the variance in results seems to stabilize. +2. You cannot really get reliable benchmarks if your host machine is simultaneously running +another cpu/memory-intensive application in the background. If you are running benchmarks on your +personal laptop for instance, you should close all applications (including your code editor and +browser) before running your benchmarks. You might also have invisible background applications +running. You can see what these are by looking at either Activity Monitor on Mac or Task Manager +on Windows. You will get more stable benchmark results of you end those processes as well. + * If you have multiple cores, you can even run your benchmark on a reserved core to prevent + pollution from other OS and user processes. There are a number of ways to do this depending + on your OS: + * On linux boxes, you have use https://github.com/lpechacek/cpuset. + * On Windows, you can "Set Processor Affinity" using https://www.thewindowsclub.com/processor-affinity-windows + * On Mac, you can try to use their dedicated affinity API https://developer.apple.com/library/archive/releasenotes/Performance/RN-AffinityAPI/#//apple_ref/doc/uid/TP40006635-CH1-DontLinkElementID_2 +3. To benchmark, you will likely end up writing a separate c/c++ program that will link libzstd. +Dynamically linking your library will introduce some added variation (not a large amount but +definitely some). Statically linking libzstd will be more stable. Static libraries should +be enabled by default when building zstd. +4. Use a profiler with a good high resolution timer. See the section below on profiling for +details on this. +5. Disable frequency scaling, turbo boost and address space randomization (this will vary by OS) +6. Try to avoid storage. On some systems you can use tmpfs. Putting the program, inputs and outputs on +tmpfs avoids touching a real storage system, which can have a pretty big variability. + +Also check our LLVM's guide on benchmarking here: https://llvm.org/docs/Benchmarking.html + +### Zstd benchmark +The fastest signal you can get regarding your performance changes is via the in-build zstd cli +bench option. You can run Zstd as you typically would for your scenario using some set of options +and then additionally also specify the `-b#` option. Doing this will run our benchmarking pipeline +for that options you have just provided. If you want to look at the internals of how this +benchmarking script works, you can check out programs/benchzstd.c + +For example: say you have made a change that you believe improves the speed of zstd level 1. The +very first thing you should use to asses whether you actually achieved any sort of improvement +is `zstd -b`. You might try to do something like this. Note: you can use the `-i` option to +specify a running time for your benchmark in seconds (default is 3 seconds). +Usually, the longer the running time, the more stable your results will be. + +``` +$ git checkout <commit-before-your-change> +$ make && cp zstd zstd-old +$ git checkout <commit-after-your-change> +$ make && cp zstd zstd-new +$ zstd-old -i5 -b1 <your-test-data> + 1<your-test-data> : 8990 -> 3992 (2.252), 302.6 MB/s , 626.4 MB/s +$ zstd-new -i5 -b1 <your-test-data> + 1<your-test-data> : 8990 -> 3992 (2.252), 302.8 MB/s , 628.4 MB/s +``` + +Unless your performance win is large enough to be visible despite the intrinsic noise +on your computer, benchzstd alone will likely not be enough to validate the impact of your +changes. For example, the results of the example above indicate that effectively nothing +changed but there could be a small <3% improvement that the noise on the host machine +obscured. So unless you see a large performance win (10-15% consistently) using just +this method of evaluation will not be sufficient. + +### Profiling +There are a number of great profilers out there. We're going to briefly mention how you can +profile your code using `instruments` on mac, `perf` on linux and `visual studio profiler` +on windows. + +Say you have an idea for a change that you think will provide some good performance gains +for level 1 compression on Zstd. Typically this means, you have identified a section of +code that you think can be made to run faster. + +The first thing you will want to do is make sure that the piece of code is actually taking up +a notable amount of time to run. It is usually not worth optimzing something which accounts for less than +0.0001% of the total running time. Luckily, there are tools to help with this. +Profilers will let you see how much time your code spends inside a particular function. +If your target code snippit is only part of a function, it might be worth trying to +isolate that snippit by moving it to its own function (this is usually not necessary but +might be). + +Most profilers (including the profilers dicusssed below) will generate a call graph of +functions for you. Your goal will be to find your function of interest in this call grapch +and then inspect the time spent inside of it. You might also want to to look at the +annotated assembly which most profilers will provide you with. + +#### Instruments +We will once again consider the scenario where you think you've identified a piece of code +whose performance can be improved upon. Follow these steps to profile your code using +Instruments. + +1. Open Instruments +2. Select `Time Profiler` from the list of standard templates +3. Close all other applications except for your instruments window and your terminal +4. Run your benchmarking script from your terminal window + * You will want a benchmark that runs for at least a few seconds (5 seconds will + usually be long enough). This way the profiler will have something to work with + and you will have ample time to attach your profiler to this process:) + * I will just use benchzstd as my bencharmking script for this example: +``` +$ zstd -b1 -i5 <my-data> # this will run for 5 seconds +``` +5. Once you run your benchmarking script, switch back over to instruments and attach your +process to the time profiler. You can do this by: + * Clicking on the `All Processes` drop down in the top left of the toolbar. + * Selecting your process from the dropdown. In my case, it is just going to be labled + `zstd` + * Hitting the bright red record circle button on the top left of the toolbar +6. You profiler will now start collecting metrics from your bencharking script. Once +you think you have collected enough samples (usually this is the case after 3 seconds of +recording), stop your profiler. +7. Make sure that in toolbar of the bottom window, `profile` is selected. +8. You should be able to see your call graph. + * If you don't see the call graph or an incomplete call graph, make sure you have compiled + zstd and your benchmarking scripg using debug flags. On mac and linux, this just means + you will have to supply the `-g` flag alone with your build script. You might also + have to provide the `-fno-omit-frame-pointer` flag +9. Dig down the graph to find your function call and then inspect it by double clicking +the list item. You will be able to see the annotated source code and the assembly side by +side. + +#### Perf + +This wiki has a pretty detailed tutorial on getting started working with perf so we'll +leave you to check that out of you're getting started: + +https://perf.wiki.kernel.org/index.php/Tutorial + +Some general notes on perf: +* Use `perf stat -r # <bench-program>` to quickly get some relevant timing and +counter statistics. Perf uses a high resolution timer and this is likely one +of the first things your team will run when assessing your PR. +* Perf has a long list of hardware counters that can be viewed with `perf --list`. +When measuring optimizations, something worth trying is to make sure the handware +counters you expect to be impacted by your change are in fact being so. For example, +if you expect the L1 cache misses to decrease with your change, you can look at the +counter `L1-dcache-load-misses` +* Perf hardware counters will not work on a virtual machine. + +#### Visual Studio + +TODO + + +## Setting up continuous integration (CI) on your fork +Zstd uses a number of different continuous integration (CI) tools to ensure that new changes +are well tested before they make it to an official release. Specifically, we use the platforms +travis-ci, circle-ci, and appveyor. + +Changes cannot be merged into the main dev branch unless they pass all of our CI tests. +The easiest way to run these CI tests on your own before submitting a PR to our dev branch +is to configure your personal fork of zstd with each of the CI platforms. Below, you'll find +instructions for doing this. + +### travis-ci +Follow these steps to link travis-ci with your github fork of zstd + +1. Make sure you are logged into your github account +2. Go to https://travis-ci.org/ +3. Click 'Sign in with Github' on the top right +4. Click 'Authorize travis-ci' +5. Click 'Activate all repositories using Github Apps' +6. Select 'Only select repositories' and select your fork of zstd from the drop down +7. Click 'Approve and Install' +8. Click 'Sign in with Github' again. This time, it will be for travis-pro (which will let you view your tests on the web dashboard) +9. Click 'Authorize travis-pro' +10. You should have travis set up on your fork now. + +### circle-ci +TODO + +### appveyor +Follow these steps to link circle-ci with your girhub fork of zstd + +1. Make sure you are logged into your github account +2. Go to https://www.appveyor.com/ +3. Click 'Sign in' on the top right +4. Select 'Github' on the left panel +5. Click 'Authorize appveyor' +6. You might be asked to select which repositories you want to give appveyor permission to. Select your fork of zstd if you're prompted +7. You should have appveyor set up on your fork now. + +### General notes on CI +CI tests run every time a pull request (PR) is created or updated. The exact tests +that get run will depend on the destination branch you specify. Some tests take +longer to run than others. Currently, our CI is set up to run a short +series of tests when creating a PR to the dev branch and a longer series of tests +when creating a PR to the master branch. You can look in the configuration files +of the respective CI platform for more information on what gets run when. + +Most people will just want to create a PR with the destination set to their local dev +branch of zstd. You can then find the status of the tests on the PR's page. You can also +re-run tests and cancel running tests from the PR page or from the respective CI's dashboard. + ## Issues We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. @@ -34,7 +384,7 @@ Facebook has a [bounty program](https://www.facebook.c disclosure of security bugs. In those cases, please go through the process outlined on that page and do not file a public issue. -## Coding Style +## Coding Style * 4 spaces for indentation rather than tabs ## License Modified: head/sys/contrib/zstd/Makefile ============================================================================== --- head/sys/contrib/zstd/Makefile Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/Makefile Sat May 23 21:23:46 2020 (r361426) @@ -1,10 +1,11 @@ # ################################################################ -# Copyright (c) 2015-present, Yann Collet, Facebook, Inc. +# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the # LICENSE file in the root directory of this source tree) and the GPLv2 (found # in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. # ################################################################ PRGDIR = programs @@ -17,7 +18,16 @@ FUZZDIR = $(TESTDIR)/fuzz # Define nul output VOID = /dev/null -ifneq (,$(filter Windows%,$(OS))) +# When cross-compiling from linux to windows, you might +# need to specify this as "Windows." Fedora build fails +# without it. +# +# Note: mingw-w64 build from linux to windows does not +# fail on other tested distros (ubuntu, debian) even +# without manually specifying the TARGET_SYSTEM. +TARGET_SYSTEM ?= $(OS) + +ifneq (,$(filter Windows%,$(TARGET_SYSTEM))) EXT =.exe else EXT = @@ -35,7 +45,7 @@ allmost: allzstd zlibwrapper # skip zwrapper, can't build that on alternate architectures without the proper zlib installed .PHONY: allzstd -allzstd: lib +allzstd: lib-all $(MAKE) -C $(PRGDIR) all $(MAKE) -C $(TESTDIR) all @@ -45,7 +55,7 @@ all32: $(MAKE) -C $(TESTDIR) all32 .PHONY: lib lib-release libzstd.a -lib lib-release : +lib lib-release lib-all : @$(MAKE) -C $(ZSTDDIR) $@ .PHONY: zstd zstd-release @@ -80,6 +90,13 @@ shortest: .PHONY: check check: shortest +.PHONY: automated_benchmarking +automated_benchmarking: + $(MAKE) -C $(TESTDIR) $@ + +.PHONY: benchmarking +benchmarking: automated_benchmarking + ## examples: build all examples in `/examples` directory .PHONY: examples examples: lib @@ -101,7 +118,8 @@ contrib: lib $(MAKE) -C contrib/pzstd all $(MAKE) -C contrib/seekable_format/examples all $(MAKE) -C contrib/largeNbDicts all - cd contrib/single_file_decoder/ ; ./build_test.sh + cd contrib/single_file_libs/ ; ./build_decoder_test.sh + cd contrib/single_file_libs/ ; ./build_library_test.sh .PHONY: cleanTabs cleanTabs: @@ -337,7 +355,7 @@ endif ifneq (,$(filter MSYS%,$(shell uname))) HOST_OS = MSYS -CMAKE_PARAMS = -G"MSYS Makefiles" -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON +CMAKE_PARAMS = -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Debug -DZSTD_MULTITHREAD_SUPPORT:BOOL=OFF -DZSTD_BUILD_STATIC:BOOL=ON -DZSTD_BUILD_TESTS:BOOL=ON endif @@ -349,11 +367,15 @@ cmakebuild: cmake --version $(RM) -r $(BUILDIR)/cmake/build mkdir $(BUILDIR)/cmake/build - cd $(BUILDIR)/cmake/build ; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. ; $(MAKE) install ; $(MAKE) uninstall + cd $(BUILDIR)/cmake/build; cmake -DCMAKE_INSTALL_PREFIX:PATH=~/install_test_dir $(CMAKE_PARAMS) .. + $(MAKE) -C $(BUILDIR)/cmake/build -j4; + $(MAKE) -C $(BUILDIR)/cmake/build install; + $(MAKE) -C $(BUILDIR)/cmake/build uninstall; + cd $(BUILDIR)/cmake/build; ctest -V -L Medium -c90build: clean +c89build: clean $(CC) -v - CFLAGS="-std=c90 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long` + CFLAGS="-std=c89 -Werror" $(MAKE) allmost # will fail, due to missing support for `long long` gnu90build: clean $(CC) -v Modified: head/sys/contrib/zstd/README.md ============================================================================== --- head/sys/contrib/zstd/README.md Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/README.md Sat May 23 21:23:46 2020 (r361426) @@ -31,10 +31,10 @@ a list of known ports and bindings is provided on [Zst ## Benchmarks For reference, several fast compression algorithms were tested and compared -on a server running Arch Linux (`Linux version 5.0.5-arch1-1`), +on a server running Arch Linux (`Linux version 5.5.11-arch1-1`), with a Core i9-9900K CPU @ 5.0GHz, using [lzbench], an open-source in-memory benchmark by @inikep -compiled with [gcc] 8.2.1, +compiled with [gcc] 9.3.0, on the [Silesia compression corpus]. [lzbench]: https://github.com/inikep/lzbench @@ -43,18 +43,26 @@ on the [Silesia compression corpus]. | Compressor name | Ratio | Compression| Decompress.| | --------------- | ------| -----------| ---------- | -| **zstd 1.4.0 -1** | 2.884 | 530 MB/s | 1360 MB/s | -| zlib 1.2.11 -1 | 2.743 | 110 MB/s | 440 MB/s | -| brotli 1.0.7 -0 | 2.701 | 430 MB/s | 470 MB/s | -| quicklz 1.5.0 -1 | 2.238 | 600 MB/s | 800 MB/s | -| lzo1x 2.09 -1 | 2.106 | 680 MB/s | 950 MB/s | -| lz4 1.8.3 | 2.101 | 800 MB/s | 4220 MB/s | -| snappy 1.1.4 | 2.073 | 580 MB/s | 2020 MB/s | -| lzf 3.6 -1 | 2.077 | 440 MB/s | 930 MB/s | +| **zstd 1.4.5 -1** | 2.884 | 500 MB/s | 1660 MB/s | +| zlib 1.2.11 -1 | 2.743 | 90 MB/s | 400 MB/s | +| brotli 1.0.7 -0 | 2.703 | 400 MB/s | 450 MB/s | +| **zstd 1.4.5 --fast=1** | 2.434 | 570 MB/s | 2200 MB/s | +| **zstd 1.4.5 --fast=3** | 2.312 | 640 MB/s | 2300 MB/s | +| quicklz 1.5.0 -1 | 2.238 | 560 MB/s | 710 MB/s | +| **zstd 1.4.5 --fast=5** | 2.178 | 700 MB/s | 2420 MB/s | +| lzo1x 2.10 -1 | 2.106 | 690 MB/s | 820 MB/s | +| lz4 1.9.2 | 2.101 | 740 MB/s | 4530 MB/s | +| **zstd 1.4.5 --fast=7** | 2.096 | 750 MB/s | 2480 MB/s | +| lzf 3.6 -1 | 2.077 | 410 MB/s | 860 MB/s | +| snappy 1.1.8 | 2.073 | 560 MB/s | 1790 MB/s | [zlib]: http://www.zlib.net/ [LZ4]: http://www.lz4.org/ +The negative compression levels, specified with `--fast=#`, +offer faster compression and decompression speed in exchange for some loss in +compression ratio compared to level 1, as seen in the table above. + Zstd can also offer stronger compression ratios at the cost of compression speed. Speed vs Compression trade-off is configurable by small increments. Decompression speed is preserved and remains roughly the same at all settings, @@ -142,6 +150,18 @@ You can also take a look at [`.travis.yml`](.travis.ym example about how Meson is used to build this project. Note that default build type is **release**. + +### VCPKG +You can build and install zstd [vcpkg](https://github.com/Microsoft/vcpkg/) dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install zstd + +The zstd port in vcpkg is kept up to date by Microsoft team members and community contributors. +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. ### Visual Studio (Windows) Modified: head/sys/contrib/zstd/TESTING.md ============================================================================== --- head/sys/contrib/zstd/TESTING.md Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/TESTING.md Sat May 23 21:23:46 2020 (r361426) @@ -11,7 +11,7 @@ They consist of the following tests: - Compilation on all supported targets (x86, x86_64, ARM, AArch64, PowerPC, and PowerPC64) - Compilation on various versions of gcc, clang, and g++ - `tests/playTests.sh` on x86_64, without the tests on long data (CLI tests) -- Small tests (`tests/legacy.c`, `tests/longmatch.c`, `tests/symbols.c`) on x64_64 +- Small tests (`tests/legacy.c`, `tests/longmatch.c`) on x64_64 Medium Tests ------------ @@ -19,7 +19,7 @@ Medium tests run on every commit and pull request to ` They consist of the following tests: - The following tests run with UBsan and Asan on x86_64 and x86, as well as with Msan on x86_64 - - `tests/playTests.sh --test-long-data` + - `tests/playTests.sh --test-large-data` - Fuzzer tests: `tests/fuzzer.c`, `tests/zstreamtest.c`, and `tests/decodecorpus.c` - `tests/zstreamtest.c` under Tsan (streaming mode, including multithreaded mode) - Valgrind Test (`make -C tests valgrindTest`) (testing CLI and fuzzer under valgrind) Modified: head/sys/contrib/zstd/appveyor.yml ============================================================================== --- head/sys/contrib/zstd/appveyor.yml Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/appveyor.yml Sat May 23 21:23:46 2020 (r361426) @@ -14,7 +14,7 @@ - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" - SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests test-symbols fullbench-lib" + SCRIPT: "make allzstd MOREFLAGS=-static && make -C tests fullbench-lib" ARTIFACT: "true" BUILD: "true" - COMPILER: "gcc" @@ -169,7 +169,8 @@ - SET "FUZZERTEST=-T30s" - if [%HOST%]==[visual] if [%CONFIGURATION%]==[Release] ( CD tests && - SET ZSTD=./zstd.exe && + SET ZSTD_BIN=./zstd.exe&& + SET DATAGEN_BIN=./datagen.exe&& sh -e playTests.sh --test-large-data && fullbench.exe -i1 && fullbench.exe -i1 -P0 && @@ -188,6 +189,9 @@ environment: matrix: - COMPILER: "gcc" + HOST: "cygwin" + PLATFORM: "x64" + - COMPILER: "gcc" HOST: "mingw" PLATFORM: "x64" SCRIPT: "CPPFLAGS=-DDEBUGLEVEL=2 CFLAGS=-Werror make -j allzstd DEBUGLEVEL=2" @@ -220,6 +224,14 @@ install: - ECHO Installing %COMPILER% %PLATFORM% %CONFIGURATION% - SET PATH_ORIGINAL=%PATH% + - if [%HOST%]==[cygwin] ( + ECHO Installing Cygwin Packages && + C:\cygwin64\setup-x86_64.exe -qnNdO -R "C:\cygwin64" -g -P ^ + gcc-g++,^ + gcc,^ + cmake,^ + make + ) - if [%HOST%]==[mingw] ( SET "PATH_MINGW32=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin" && SET "PATH_MINGW64=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin" && @@ -232,6 +244,17 @@ build_script: - ECHO Building %COMPILER% %PLATFORM% %CONFIGURATION% + - if [%HOST%]==[cygwin] ( + set CHERE_INVOKING=yes && + set CC=%COMPILER% && + C:\cygwin64\bin\bash --login -c " + set -e; + cd build/cmake; + CFLAGS='-Werror' cmake -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=Debug -DZSTD_BUILD_TESTS:BOOL=ON -DZSTD_FUZZER_FLAGS=-T30s -DZSTD_ZSTREAM_FLAGS=-T30s .; + make -j4; + ctest -V -L Medium; + " + ) - if [%HOST%]==[mingw] ( ( if [%PLATFORM%]==[x64] ( SET "PATH=%PATH_MINGW64%;%PATH_ORIGINAL%" Modified: head/sys/contrib/zstd/doc/educational_decoder/Makefile ============================================================================== --- head/sys/contrib/zstd/doc/educational_decoder/Makefile Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/doc/educational_decoder/Makefile Sat May 23 21:23:46 2020 (r361426) @@ -1,10 +1,11 @@ # ################################################################ -# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. # All rights reserved. # # This source code is licensed under both the BSD-style license (found in the # LICENSE file in the root directory of this source tree) and the GPLv2 (found # in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. # ################################################################ ZSTD ?= zstd # note: requires zstd installation on local system @@ -36,7 +37,7 @@ harness: $(HARNESS_FILES) $(CC) $(FLAGS) $^ -o $@ clean: - @$(RM) harness + @$(RM) harness *.o @$(RM) -rf harness.dSYM # MacOS specific test: harness @@ -59,4 +60,3 @@ test: harness @./harness tmp.zst tmp dictionary @$(DIFF) -s tmp README.md @$(RM) tmp* dictionary - @$(MAKE) clean Modified: head/sys/contrib/zstd/doc/educational_decoder/README.md ============================================================================== --- head/sys/contrib/zstd/doc/educational_decoder/README.md Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/doc/educational_decoder/README.md Sat May 23 21:23:46 2020 (r361426) @@ -13,6 +13,13 @@ It also contains implementations of Huffman and FSE ta [Zstandard format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md [format specification]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md +While the library's primary objective is code clarity, +it also happens to compile into a small object file. +The object file can be made even smaller by removing error messages, +using the macro directive `ZDEC_NO_MESSAGE` at compilation time. +This can be reduced even further by foregoing dictionary support, +by defining `ZDEC_NO_DICTIONARY`. + `harness.c` provides a simple test harness around the decoder: harness <input-file> <output-file> [dictionary] Modified: head/sys/contrib/zstd/doc/educational_decoder/harness.c ============================================================================== --- head/sys/contrib/zstd/doc/educational_decoder/harness.c Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/doc/educational_decoder/harness.c Sat May 23 21:23:46 2020 (r361426) @@ -1,10 +1,11 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) 2017-2020, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ #include <stdio.h> @@ -21,108 +22,98 @@ typedef unsigned char u8; // Protect against allocating too much memory for output #define MAX_OUTPUT_SIZE ((size_t)1024 * 1024 * 1024) -static size_t read_file(const char *path, u8 **ptr) +// Error message then exit +#define ERR_OUT(...) { fprintf(stderr, __VA_ARGS__); exit(1); } + + +typedef struct { + u8* address; + size_t size; +} buffer_s; + +static void freeBuffer(buffer_s b) { free(b.address); } + +static buffer_s read_file(const char *path) { FILE* const f = fopen(path, "rb"); - if (!f) { - fprintf(stderr, "failed to open file %s \n", path); - exit(1); - } + if (!f) ERR_OUT("failed to open file %s \n", path); fseek(f, 0L, SEEK_END); size_t const size = (size_t)ftell(f); rewind(f); - *ptr = malloc(size); - if (!ptr) { - fprintf(stderr, "failed to allocate memory to hold %s \n", path); - exit(1); - } + void* const ptr = malloc(size); + if (!ptr) ERR_OUT("failed to allocate memory to hold %s \n", path); - size_t const read = fread(*ptr, 1, size, f); - if (read != size) { /* must read everything in one pass */ - fprintf(stderr, "error while reading file %s \n", path); - exit(1); - } + size_t const read = fread(ptr, 1, size, f); + if (read != size) ERR_OUT("error while reading file %s \n", path); fclose(f); - - return read; + buffer_s const b = { ptr, size }; + return b; } -static void write_file(const char *path, const u8 *ptr, size_t size) +static void write_file(const char* path, const u8* ptr, size_t size) { FILE* const f = fopen(path, "wb"); - if (!f) { - fprintf(stderr, "failed to open file %s \n", path); - exit(1); - } + if (!f) ERR_OUT("failed to open file %s \n", path); size_t written = 0; while (written < size) { written += fwrite(ptr+written, 1, size, f); - if (ferror(f)) { - fprintf(stderr, "error while writing file %s\n", path); - exit(1); - } } + if (ferror(f)) ERR_OUT("error while writing file %s\n", path); + } fclose(f); } int main(int argc, char **argv) { - if (argc < 3) { - fprintf(stderr, "usage: %s <file.zst> <out_path> [dictionary] \n", - argv[0]); + if (argc < 3) + ERR_OUT("usage: %s <file.zst> <out_path> [dictionary] \n", argv[0]); - return 1; - } + buffer_s const input = read_file(argv[1]); - u8* input; - size_t const input_size = read_file(argv[1], &input); - - u8* dict = NULL; - size_t dict_size = 0; + buffer_s dict = { NULL, 0 }; if (argc >= 4) { - dict_size = read_file(argv[3], &dict); + dict = read_file(argv[3]); } - size_t out_capacity = ZSTD_get_decompressed_size(input, input_size); + size_t out_capacity = ZSTD_get_decompressed_size(input.address, input.size); if (out_capacity == (size_t)-1) { - out_capacity = MAX_COMPRESSION_RATIO * input_size; + out_capacity = MAX_COMPRESSION_RATIO * input.size; fprintf(stderr, "WARNING: Compressed data does not contain " "decompressed size, going to assume the compression " "ratio is at most %d (decompressed size of at most " "%u) \n", MAX_COMPRESSION_RATIO, (unsigned)out_capacity); } - if (out_capacity > MAX_OUTPUT_SIZE) { - fprintf(stderr, - "Required output size too large for this implementation \n"); - return 1; - } + if (out_capacity > MAX_OUTPUT_SIZE) + ERR_OUT("Required output size too large for this implementation \n"); u8* const output = malloc(out_capacity); - if (!output) { - fprintf(stderr, "failed to allocate memory \n"); - return 1; - } + if (!output) ERR_OUT("failed to allocate memory \n"); dictionary_t* const parsed_dict = create_dictionary(); - if (dict) { - parse_dictionary(parsed_dict, dict, dict_size); + if (dict.size) { +#if defined (ZDEC_NO_DICTIONARY) + printf("dict.size = %zu \n", dict.size); + ERR_OUT("no dictionary support \n"); +#else + parse_dictionary(parsed_dict, dict.address, dict.size); +#endif } size_t const decompressed_size = ZSTD_decompress_with_dict(output, out_capacity, - input, input_size, + input.address, input.size, parsed_dict); free_dictionary(parsed_dict); write_file(argv[2], output, decompressed_size); - free(input); + freeBuffer(input); + freeBuffer(dict); free(output); - free(dict); return 0; } Modified: head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c ============================================================================== --- head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c Sat May 23 20:39:36 2020 (r361425) +++ head/sys/contrib/zstd/doc/educational_decoder/zstd_decompress.c Sat May 23 21:23:46 2020 (r361426) @@ -1,34 +1,52 @@ /* - * Copyright (c) 2017-present, Facebook, Inc. + * Copyright (c) 2017-2020, Facebook, Inc. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the * LICENSE file in the root directory of this source tree) and the GPLv2 (found * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. */ /// Zstandard educational decoder implementation /// See https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> +#include <stdint.h> // uint8_t, etc. +#include <stdlib.h> // malloc, free, exit +#include <stdio.h> // fprintf +#include <string.h> // memset, memcpy #include "zstd_decompress.h" -/******* UTILITY MACROS AND TYPES *********************************************/ -// Max block size decompressed size is 128 KB and literal blocks can't be -// larger than their block -#define MAX_LITERALS_SIZE ((size_t)128 * 1024) +/******* IMPORTANT CONSTANTS *********************************************/ + +// Zstandard frame +// "Magic_Number +// 4 Bytes, little-endian format. Value : 0xFD2FB528" +#define ZSTD_MAGIC_NUMBER 0xFD2FB528U + +// The size of `Block_Content` is limited by `Block_Maximum_Size`, +#define ZSTD_BLOCK_SIZE_MAX ((size_t)128 * 1024) + +// literal blocks can't be larger than their block +#define MAX_LITERALS_SIZE ZSTD_BLOCK_SIZE_MAX + + +/******* UTILITY MACROS AND TYPES *********************************************/ #define MAX(a, b) ((a) > (b) ? (a) : (b)) #define MIN(a, b) ((a) < (b) ? (a) : (b)) +#if defined(ZDEC_NO_MESSAGE) +#define MESSAGE(...) +#else +#define MESSAGE(...) fprintf(stderr, "" __VA_ARGS__) +#endif + /// This decoder calls exit(1) when it encounters an error, however a production /// library should propagate error codes #define ERROR(s) \ do { \ - fprintf(stderr, "Error: %s\n", s); \ + MESSAGE("Error: %s\n", s); \ exit(1); \ } while (0) #define INP_SIZE() \ @@ -39,12 +57,12 @@ #define BAD_ALLOC() ERROR("Memory allocation error") #define IMPOSSIBLE() ERROR("An impossibility has occurred") -typedef uint8_t u8; +typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; -typedef int8_t i8; +typedef int8_t i8; typedef int16_t i16; typedef int32_t i32; typedef int64_t i64; @@ -176,10 +194,6 @@ static void HUF_init_dtable_usingweights(HUF_dtable *c /// Free the malloc'ed parts of a decoding table static void HUF_free_dtable(HUF_dtable *const dtable); - -/// Deep copy a decoding table, so that it can be used and free'd without -/// impacting the source table. -static void HUF_copy_dtable(HUF_dtable *const dst, const HUF_dtable *const src); *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202005232123.04NLNkG4078947>