Date: Mon, 11 Nov 2019 03:27:14 +0000 (UTC) From: Jason Evans <jasone@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r354605 - in head: contrib/jemalloc contrib/jemalloc/doc contrib/jemalloc/include/jemalloc contrib/jemalloc/include/jemalloc/internal contrib/jemalloc/src lib/libc/stdlib/jemalloc Message-ID: <201911110327.xAB3REbS098393@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: jasone Date: Mon Nov 11 03:27:14 2019 New Revision: 354605 URL: https://svnweb.freebsd.org/changeset/base/354605 Log: Update jemalloc to version 5.2.1. Added: head/contrib/jemalloc/include/jemalloc/internal/bin_types.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/hook.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/quantum.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/safety_check.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/sc.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/seq.h (contents, props changed) head/contrib/jemalloc/include/jemalloc/internal/test_hooks.h (contents, props changed) head/contrib/jemalloc/src/hook.c (contents, props changed) head/contrib/jemalloc/src/safety_check.c (contents, props changed) head/contrib/jemalloc/src/sc.c (contents, props changed) head/contrib/jemalloc/src/test_hooks.c (contents, props changed) Deleted: head/contrib/jemalloc/include/jemalloc/internal/hooks.h head/contrib/jemalloc/include/jemalloc/internal/size_classes.h head/contrib/jemalloc/src/hooks.c Modified: head/contrib/jemalloc/COPYING head/contrib/jemalloc/ChangeLog head/contrib/jemalloc/FREEBSD-Xlist head/contrib/jemalloc/FREEBSD-diffs head/contrib/jemalloc/VERSION head/contrib/jemalloc/doc/jemalloc.3 head/contrib/jemalloc/include/jemalloc/internal/arena_externs.h head/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h head/contrib/jemalloc/include/jemalloc/internal/arena_stats.h head/contrib/jemalloc/include/jemalloc/internal/arena_structs_b.h head/contrib/jemalloc/include/jemalloc/internal/arena_types.h head/contrib/jemalloc/include/jemalloc/internal/atomic.h head/contrib/jemalloc/include/jemalloc/internal/atomic_gcc_atomic.h head/contrib/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h head/contrib/jemalloc/include/jemalloc/internal/background_thread_externs.h head/contrib/jemalloc/include/jemalloc/internal/background_thread_inlines.h head/contrib/jemalloc/include/jemalloc/internal/background_thread_structs.h head/contrib/jemalloc/include/jemalloc/internal/base_structs.h head/contrib/jemalloc/include/jemalloc/internal/bin.h head/contrib/jemalloc/include/jemalloc/internal/bin_stats.h head/contrib/jemalloc/include/jemalloc/internal/bit_util.h head/contrib/jemalloc/include/jemalloc/internal/bitmap.h head/contrib/jemalloc/include/jemalloc/internal/cache_bin.h head/contrib/jemalloc/include/jemalloc/internal/ctl.h head/contrib/jemalloc/include/jemalloc/internal/emitter.h head/contrib/jemalloc/include/jemalloc/internal/extent_externs.h head/contrib/jemalloc/include/jemalloc/internal/extent_inlines.h head/contrib/jemalloc/include/jemalloc/internal/extent_structs.h head/contrib/jemalloc/include/jemalloc/internal/extent_types.h head/contrib/jemalloc/include/jemalloc/internal/hash.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h head/contrib/jemalloc/include/jemalloc/internal/jemalloc_preamble.h head/contrib/jemalloc/include/jemalloc/internal/large_externs.h head/contrib/jemalloc/include/jemalloc/internal/malloc_io.h head/contrib/jemalloc/include/jemalloc/internal/mutex.h head/contrib/jemalloc/include/jemalloc/internal/mutex_prof.h head/contrib/jemalloc/include/jemalloc/internal/private_namespace.h head/contrib/jemalloc/include/jemalloc/internal/prof_externs.h head/contrib/jemalloc/include/jemalloc/internal/prof_inlines_a.h head/contrib/jemalloc/include/jemalloc/internal/prof_inlines_b.h head/contrib/jemalloc/include/jemalloc/internal/prof_structs.h head/contrib/jemalloc/include/jemalloc/internal/public_namespace.h head/contrib/jemalloc/include/jemalloc/internal/rtree.h head/contrib/jemalloc/include/jemalloc/internal/rtree_tsd.h head/contrib/jemalloc/include/jemalloc/internal/stats.h head/contrib/jemalloc/include/jemalloc/internal/sz.h head/contrib/jemalloc/include/jemalloc/internal/tcache_externs.h head/contrib/jemalloc/include/jemalloc/internal/tcache_inlines.h head/contrib/jemalloc/include/jemalloc/internal/tcache_structs.h head/contrib/jemalloc/include/jemalloc/internal/tcache_types.h head/contrib/jemalloc/include/jemalloc/internal/ticker.h head/contrib/jemalloc/include/jemalloc/internal/tsd.h head/contrib/jemalloc/include/jemalloc/internal/tsd_generic.h head/contrib/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h head/contrib/jemalloc/include/jemalloc/internal/tsd_tls.h head/contrib/jemalloc/include/jemalloc/internal/witness.h head/contrib/jemalloc/include/jemalloc/jemalloc.h head/contrib/jemalloc/src/arena.c head/contrib/jemalloc/src/background_thread.c head/contrib/jemalloc/src/base.c head/contrib/jemalloc/src/bin.c head/contrib/jemalloc/src/ckh.c head/contrib/jemalloc/src/ctl.c head/contrib/jemalloc/src/extent.c head/contrib/jemalloc/src/extent_dss.c head/contrib/jemalloc/src/extent_mmap.c head/contrib/jemalloc/src/jemalloc.c head/contrib/jemalloc/src/large.c head/contrib/jemalloc/src/malloc_io.c head/contrib/jemalloc/src/mutex.c head/contrib/jemalloc/src/pages.c head/contrib/jemalloc/src/prof.c head/contrib/jemalloc/src/rtree.c head/contrib/jemalloc/src/stats.c head/contrib/jemalloc/src/sz.c head/contrib/jemalloc/src/tcache.c head/contrib/jemalloc/src/tsd.c head/lib/libc/stdlib/jemalloc/Makefile.inc Modified: head/contrib/jemalloc/COPYING ============================================================================== --- head/contrib/jemalloc/COPYING Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/COPYING Mon Nov 11 03:27:14 2019 (r354605) @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2018 Jason Evans <jasone@canonware.com>. +Copyright (C) 2002-present Jason Evans <jasone@canonware.com>. All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2018 Facebook, Inc. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Modified: head/contrib/jemalloc/ChangeLog ============================================================================== --- head/contrib/jemalloc/ChangeLog Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/ChangeLog Mon Nov 11 03:27:14 2019 (r354605) @@ -4,7 +4,143 @@ brevity. Much more detail can be found in the git rev https://github.com/jemalloc/jemalloc -* 5.1.0 (May 4th, 2018) +* 5.2.1 (August 5, 2019) + + This release is primarily about Windows. A critical virtual memory leak is + resolved on all Windows platforms. The regression was present in all releases + since 5.0.0. + + Bug fixes: + - Fix a severe virtual memory leak on Windows. This regression was first + released in 5.0.0. (@Ignition, @j0t, @frederik-h, @davidtgoldblatt, + @interwq) + - Fix size 0 handling in posix_memalign(). This regression was first released + in 5.2.0. (@interwq) + - Fix the prof_log unit test which may observe unexpected backtraces from + compiler optimizations. The test was first added in 5.2.0. (@marxin, + @gnzlbg, @interwq) + - Fix the declaration of the extent_avail tree. This regression was first + released in 5.1.0. (@zoulasc) + - Fix an incorrect reference in jeprof. This functionality was first released + in 3.0.0. (@prehistoric-penguin) + - Fix an assertion on the deallocation fast-path. This regression was first + released in 5.2.0. (@yinan1048576) + - Fix the TLS_MODEL attribute in headers. This regression was first released + in 5.0.0. (@zoulasc, @interwq) + + Optimizations and refactors: + - Implement opt.retain on Windows and enable by default on 64-bit. (@interwq, + @davidtgoldblatt) + - Optimize away a branch on the operator delete[] path. (@mgrice) + - Add format annotation to the format generator function. (@zoulasc) + - Refactor and improve the size class header generation. (@yinan1048576) + - Remove best fit. (@djwatson) + - Avoid blocking on background thread locks for stats. (@oranagra, @interwq) + +* 5.2.0 (April 2, 2019) + + This release includes a few notable improvements, which are summarized below: + 1) improved fast-path performance from the optimizations by @djwatson; 2) + reduced virtual memory fragmentation and metadata usage; and 3) bug fixes on + setting the number of background threads. In addition, peak / spike memory + usage is improved with certain allocation patterns. As usual, the release and + prior dev versions have gone through large-scale production testing. + + New features: + - Implement oversize_threshold, which uses a dedicated arena for allocations + crossing the specified threshold to reduce fragmentation. (@interwq) + - Add extents usage information to stats. (@tyleretzel) + - Log time information for sampled allocations. (@tyleretzel) + - Support 0 size in sdallocx. (@djwatson) + - Output rate for certain counters in malloc_stats. (@zinoale) + - Add configure option --enable-readlinkat, which allows the use of readlinkat + over readlink. (@davidtgoldblatt) + - Add configure options --{enable,disable}-{static,shared} to allow not + building unwanted libraries. (@Ericson2314) + - Add configure option --disable-libdl to enable fully static builds. + (@interwq) + - Add mallctl interfaces: + + opt.oversize_threshold (@interwq) + + stats.arenas.<i>.extent_avail (@tyleretzel) + + stats.arenas.<i>.extents.<j>.n{dirty,muzzy,retained} (@tyleretzel) + + stats.arenas.<i>.extents.<j>.{dirty,muzzy,retained}_bytes + (@tyleretzel) + + Portability improvements: + - Update MSVC builds. (@maksqwe, @rustyx) + - Workaround a compiler optimizer bug on s390x. (@rkmisra) + - Make use of pthread_set_name_np(3) on FreeBSD. (@trasz) + - Implement malloc_getcpu() to enable percpu_arena for windows. (@santagada) + - Link against -pthread instead of -lpthread. (@paravoid) + - Make background_thread not dependent on libdl. (@interwq) + - Add stringify to fix a linker directive issue on MSVC. (@daverigby) + - Detect and fall back when 8-bit atomics are unavailable. (@interwq) + - Fall back to the default pthread_create if dlsym(3) fails. (@interwq) + + Optimizations and refactors: + - Refactor the TSD module. (@davidtgoldblatt) + - Avoid taking extents_muzzy mutex when muzzy is disabled. (@interwq) + - Avoid taking large_mtx for auto arenas on the tcache flush path. (@interwq) + - Optimize ixalloc by avoiding a size lookup. (@interwq) + - Implement opt.oversize_threshold which uses a dedicated arena for requests + crossing the threshold, also eagerly purges the oversize extents. Default + the threshold to 8 MiB. (@interwq) + - Clean compilation with -Wextra. (@gnzlbg, @jasone) + - Refactor the size class module. (@davidtgoldblatt) + - Refactor the stats emitter. (@tyleretzel) + - Optimize pow2_ceil. (@rkmisra) + - Avoid runtime detection of lazy purging on FreeBSD. (@trasz) + - Optimize mmap(2) alignment handling on FreeBSD. (@trasz) + - Improve error handling for THP state initialization. (@jsteemann) + - Rework the malloc() fast path. (@djwatson) + - Rework the free() fast path. (@djwatson) + - Refactor and optimize the tcache fill / flush paths. (@djwatson) + - Optimize sync / lwsync on PowerPC. (@chmeeedalf) + - Bypass extent_dalloc() when retain is enabled. (@interwq) + - Optimize the locking on large deallocation. (@interwq) + - Reduce the number of pages committed from sanity checking in debug build. + (@trasz, @interwq) + - Deprecate OSSpinLock. (@interwq) + - Lower the default number of background threads to 4 (when the feature + is enabled). (@interwq) + - Optimize the trylock spin wait. (@djwatson) + - Use arena index for arena-matching checks. (@interwq) + - Avoid forced decay on thread termination when using background threads. + (@interwq) + - Disable muzzy decay by default. (@djwatson, @interwq) + - Only initialize libgcc unwinder when profiling is enabled. (@paravoid, + @interwq) + + Bug fixes (all only relevant to jemalloc 5.x): + - Fix background thread index issues with max_background_threads. (@djwatson, + @interwq) + - Fix stats output for opt.lg_extent_max_active_fit. (@interwq) + - Fix opt.prof_prefix initialization. (@davidtgoldblatt) + - Properly trigger decay on tcache destroy. (@interwq, @amosbird) + - Fix tcache.flush. (@interwq) + - Detect whether explicit extent zero out is necessary with huge pages or + custom extent hooks, which may change the purge semantics. (@interwq) + - Fix a side effect caused by extent_max_active_fit combined with decay-based + purging, where freed extents can accumulate and not be reused for an + extended period of time. (@interwq, @mpghf) + - Fix a missing unlock on extent register error handling. (@zoulasc) + + Testing: + - Simplify the Travis script output. (@gnzlbg) + - Update the test scripts for FreeBSD. (@devnexen) + - Add unit tests for the producer-consumer pattern. (@interwq) + - Add Cirrus-CI config for FreeBSD builds. (@jasone) + - Add size-matching sanity checks on tcache flush. (@davidtgoldblatt, + @interwq) + + Incompatible changes: + - Remove --with-lg-page-sizes. (@davidtgoldblatt) + + Documentation: + - Attempt to build docs by default, however skip doc building when xsltproc + is missing. (@interwq, @cmuellner) + +* 5.1.0 (May 4, 2018) This release is primarily about fine-tuning, ranging from several new features to numerous notable performance and portability enhancements. The release and Modified: head/contrib/jemalloc/FREEBSD-Xlist ============================================================================== --- head/contrib/jemalloc/FREEBSD-Xlist Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/FREEBSD-Xlist Mon Nov 11 03:27:14 2019 (r354605) @@ -1,6 +1,7 @@ $FreeBSD$ .appveyor.yml .autom4te.cfg +.cirrus.yml .git* .travis.yml FREEBSD-* Modified: head/contrib/jemalloc/FREEBSD-diffs ============================================================================== --- head/contrib/jemalloc/FREEBSD-diffs Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/FREEBSD-diffs Mon Nov 11 03:27:14 2019 (r354605) @@ -1,5 +1,5 @@ diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in -index 1e12fd3a..c42a7e10 100644 +index 7fecda7c..d5ca5e86 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -53,11 +53,22 @@ @@ -26,7 +26,7 @@ index 1e12fd3a..c42a7e10 100644 <refsect2> <title>Standard API</title> <funcprototype> -@@ -3376,4 +3387,18 @@ malloc_conf = "narenas:1";]]></programlisting></para> +@@ -3510,4 +3521,18 @@ malloc_conf = "narenas:1";]]></programlisting></para> <para>The <function>posix_memalign()</function> function conforms to IEEE Std 1003.1-2001 (<quote>POSIX.1</quote>).</para> </refsect1> @@ -45,26 +45,8 @@ index 1e12fd3a..c42a7e10 100644 + 11.0.</para> + </refsect1> </refentry> -diff --git a/include/jemalloc/internal/hooks.h b/include/jemalloc/internal/hooks.h -index cd49afcb..85e2a991 100644 ---- a/include/jemalloc/internal/hooks.h -+++ b/include/jemalloc/internal/hooks.h -@@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*hooks_libc_hook)(); - - #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) - --#define open JEMALLOC_HOOK(open, hooks_libc_hook) --#define read JEMALLOC_HOOK(read, hooks_libc_hook) --#define write JEMALLOC_HOOK(write, hooks_libc_hook) --#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook) --#define close JEMALLOC_HOOK(close, hooks_libc_hook) --#define creat JEMALLOC_HOOK(creat, hooks_libc_hook) --#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook) - /* Note that this is undef'd and re-define'd in src/prof.c. */ - #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) - diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h -index be70df51..84cd70da 100644 +index 7d6053e2..a0e4f5af 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -1,6 +1,9 @@ @@ -77,8 +59,23 @@ index be70df51..84cd70da 100644 #include <math.h> #ifdef _WIN32 # include <windows.h> +diff --git a/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h +new file mode 100644 +index 00000000..0dab1296 +--- /dev/null ++++ b/include/jemalloc/internal/jemalloc_internal_defs_FreeBSD.h +@@ -0,0 +1,9 @@ ++#ifndef __clang__ ++# undef JEMALLOC_INTERNAL_UNREACHABLE ++# define JEMALLOC_INTERNAL_UNREACHABLE abort ++ ++# undef JEMALLOC_C11_ATOMICS ++# undef JEMALLOC_GCC_ATOMIC_ATOMICS ++# undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS ++# undef JEMALLOC_GCC_U8_SYNC_ATOMICS ++#endif diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in -index e621fbc8..dbdd5d6b 100644 +index 3418cbfa..53e30dc4 100644 --- a/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/include/jemalloc/internal/jemalloc_preamble.h.in @@ -8,6 +8,9 @@ @@ -107,10 +104,10 @@ index e621fbc8..dbdd5d6b 100644 static const bool config_prof = #ifdef JEMALLOC_PROF diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h -index 6520c251..0013cbe9 100644 +index 7c24f072..94af1618 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h -@@ -121,9 +121,6 @@ struct malloc_mutex_s { +@@ -135,9 +135,6 @@ struct malloc_mutex_s { #ifdef JEMALLOC_LAZY_LOCK extern bool isthreaded; @@ -120,7 +117,7 @@ index 6520c251..0013cbe9 100644 #endif bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, -@@ -131,6 +128,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, +@@ -145,6 +142,7 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex); void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex); @@ -128,20 +125,38 @@ index 6520c251..0013cbe9 100644 bool malloc_mutex_boot(void); void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex); +diff --git a/include/jemalloc/internal/test_hooks.h b/include/jemalloc/internal/test_hooks.h +index a6351e59..0780c52f 100644 +--- a/include/jemalloc/internal/test_hooks.h ++++ b/include/jemalloc/internal/test_hooks.h +@@ -6,13 +6,6 @@ extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); + + #define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) + +-#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) +-#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) +-#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) +-#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) +-#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) +-#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) +-#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) + /* Note that this is undef'd and re-define'd in src/prof.c. */ + #define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) + diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h -index 0b9841aa..f03eee61 100644 +index 9ba26004..ecfda5d6 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h -@@ -122,7 +122,8 @@ struct tsd_s { - t use_a_getter_or_setter_instead_##n; +@@ -198,7 +198,8 @@ struct tsd_s { + t TSD_MANGLE(n); MALLOC_TSD #undef O -}; +/* AddressSanitizer requires TLS data to be aligned to at least 8 bytes. */ +} JEMALLOC_ALIGNED(16); - /* - * Wrapper around tsd_t that makes it possible to avoid implicit conversion + JEMALLOC_ALWAYS_INLINE uint8_t + tsd_state_get(tsd_t *tsd) { diff --git a/include/jemalloc/jemalloc_FreeBSD.h b/include/jemalloc/jemalloc_FreeBSD.h new file mode 100644 index 00000000..b752b0e7 @@ -345,10 +360,10 @@ index f9438912..47d032c1 100755 +#include "jemalloc_FreeBSD.h" EOF diff --git a/src/jemalloc.c b/src/jemalloc.c -index f93c16fa..e0ad297b 100644 +index ed13718d..fefb719a 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c -@@ -21,6 +21,10 @@ +@@ -23,6 +23,10 @@ /******************************************************************************/ /* Data. */ @@ -359,10 +374,37 @@ index f93c16fa..e0ad297b 100644 /* Runtime configuration options. */ const char *je_malloc_conf #ifndef _WIN32 -@@ -3160,6 +3164,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { +@@ -2660,25 +2664,6 @@ je_realloc(void *ptr, size_t arg_size) { + LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); + + if (unlikely(size == 0)) { +- if (ptr != NULL) { +- /* realloc(ptr, 0) is equivalent to free(ptr). */ +- UTRACE(ptr, 0, 0); +- tcache_t *tcache; +- tsd_t *tsd = tsd_fetch(); +- if (tsd_reentrancy_level_get(tsd) == 0) { +- tcache = tcache_get(tsd); +- } else { +- tcache = NULL; +- } +- +- uintptr_t args[3] = {(uintptr_t)ptr, size}; +- hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); +- +- ifree(tsd, ptr, tcache, true); +- +- LOG("core.realloc.exit", "result: %p", NULL); +- return NULL; +- } + size = 1; + } + +@@ -3750,6 +3735,103 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { + * End non-standard functions. */ /******************************************************************************/ - /* ++/* + * Begin compatibility functions. + */ + @@ -459,11 +501,10 @@ index f93c16fa..e0ad297b 100644 + * End compatibility functions. + */ +/******************************************************************************/ -+/* + /* * The following functions are used by threading libraries for protection of * malloc during fork(). - */ -@@ -3323,4 +3424,11 @@ jemalloc_postfork_child(void) { +@@ -3919,4 +4001,11 @@ jemalloc_postfork_child(void) { ctl_postfork_child(tsd_tsdn(tsd)); } @@ -476,7 +517,7 @@ index f93c16fa..e0ad297b 100644 + /******************************************************************************/ diff --git a/src/malloc_io.c b/src/malloc_io.c -index 7bdc13f9..c8802c70 100644 +index d7cb0f52..cda589c4 100644 --- a/src/malloc_io.c +++ b/src/malloc_io.c @@ -75,6 +75,20 @@ wrtmessage(void *cbopaque, const char *s) { @@ -501,7 +542,7 @@ index 7bdc13f9..c8802c70 100644 * Wrapper around malloc_message() that avoids the need for * je_malloc_message(...) throughout the code. diff --git a/src/mutex.c b/src/mutex.c -index 30222b3e..b2c36283 100644 +index 3f920f5b..88a7730c 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -41,6 +41,17 @@ pthread_create(pthread_t *__restrict thread, @@ -523,9 +564,10 @@ index 30222b3e..b2c36283 100644 void @@ -131,6 +142,16 @@ mutex_addr_comp(const witness_t *witness1, void *mutex1, + } } - bool ++bool +malloc_mutex_first_thread(void) { + +#ifndef JEMALLOC_MUTEX_INIT_CB @@ -535,7 +577,6 @@ index 30222b3e..b2c36283 100644 +#endif +} + -+bool + bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank, malloc_mutex_lock_order_t lock_order) { - mutex_prof_data_init(&mutex->prof_data); Modified: head/contrib/jemalloc/VERSION ============================================================================== --- head/contrib/jemalloc/VERSION Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/VERSION Mon Nov 11 03:27:14 2019 (r354605) @@ -1 +1 @@ -5.1.0-0-g61efbda7098de6fe64c362d309824864308c36d4 +5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756 Modified: head/contrib/jemalloc/doc/jemalloc.3 ============================================================================== --- head/contrib/jemalloc/doc/jemalloc.3 Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/doc/jemalloc.3 Mon Nov 11 03:27:14 2019 (r354605) @@ -1,13 +1,13 @@ '\" t .\" Title: JEMALLOC .\" Author: Jason Evans -.\" Generator: DocBook XSL Stylesheets v1.76.1 <http://docbook.sf.net/>; -.\" Date: 05/08/2018 +.\" Generator: DocBook XSL Stylesheets v1.79.1 <http://docbook.sf.net/>; +.\" Date: 11/10/2019 .\" Manual: User Manual -.\" Source: jemalloc 5.1.0-0-g61efbda7098de6fe64c362d309824864308c36d4 +.\" Source: jemalloc 5.2.1-0-gea6b3e973b477b8061e0076bb257dbd7f3faa756 .\" Language: English .\" -.TH "JEMALLOC" "3" "05/08/2018" "jemalloc 5.1.0-0-g61efbda7098d" "User Manual" +.TH "JEMALLOC" "3" "11/10/2019" "jemalloc 5.2.1-0-gea6b3e973b47" "User Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -31,7 +31,7 @@ jemalloc \- general purpose memory allocation functions .SH "LIBRARY" .PP -This manual describes jemalloc 5\&.1\&.0\-0\-g61efbda7098de6fe64c362d309824864308c36d4\&. More information can be found at the +This manual describes jemalloc 5\&.2\&.1\-0\-gea6b3e973b477b8061e0076bb257dbd7f3faa756\&. More information can be found at the \m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&. .PP The following configuration options are enabled in libc\*(Aqs built\-in jemalloc: @@ -396,7 +396,7 @@ string, in which case the statistics are presented in as a character within the \fIopts\fR string\&. Note that -malloc_message() +malloc_stats_print() uses the mallctl*() functions internally, so inconsistent statistics can be reported if multiple threads use these functions simultaneously\&. If @@ -411,7 +411,9 @@ and \(lql\(rq can be specified to omit per size class statistics for bins and large objects, respectively; \(lqx\(rq -can be specified to omit all mutex statistics\&. Unrecognized characters are silently ignored\&. Note that thread caching may prevent some statistics from being completely up to date, since extra locking would be required to merge counters that track thread cache operations\&. +can be specified to omit all mutex statistics; +\(lqe\(rq +can be used to omit extent statistics\&. Unrecognized characters are silently ignored\&. Note that thread caching may prevent some statistics from being completely up to date, since extra locking would be required to merge counters that track thread cache operations\&. .PP The malloc_usable_size() @@ -826,6 +828,17 @@ in these cases\&. This option is disabled by default u is specified during configuration, in which case it is enabled by default\&. .RE .PP +opt\&.confirm_conf (\fBbool\fR) r\- +.RS 4 +Confirm\-runtime\-options\-when\-program\-starts enabled/disabled\&. If true, the string specified via +\fB\-\-with\-malloc\-conf\fR, the string pointed to by the global variable +\fImalloc_conf\fR, the +\(lqname\(rq +of the file referenced by the symbolic link named +/etc/malloc\&.conf, and the value of the environment variable +\fBMALLOC_CONF\fR, will be printed in order\&. Then, each option being set will be individually printed\&. This option is disabled by default\&. +.RE +.PP opt\&.abort_conf (\fBbool\fR) r\- .RS 4 Abort\-on\-invalid\-configuration enabled/disabled\&. If true, invalid runtime options are fatal\&. The process will call @@ -852,9 +865,10 @@ If true, retain unused virtual memory for later reuse \fBmunmap\fR(2) or equivalent (see stats\&.retained -for related details)\&. This option is disabled by default unless discarding virtual memory is known to trigger platform\-specific performance problems, e\&.g\&. for [64\-bit] Linux, which has a quirk in its virtual memory allocation algorithm that causes semi\-permanent VM map holes under normal jemalloc operation\&. Although -\fBmunmap\fR(2) -causes issues on 32\-bit Linux as well, retaining virtual memory for 32\-bit Linux is disabled by default due to the practical possibility of address space exhaustion\&. +for related details)\&. It also makes jemalloc use +\fBmmap\fR(2) +or equivalent in a more greedy way, mapping larger chunks in one go\&. This option is disabled by default unless discarding virtual memory is known to trigger platform\-specific performance problems, namely 1) for [64\-bit] Linux, which has a quirk in its virtual memory allocation algorithm that causes semi\-permanent VM map holes under normal jemalloc operation; and 2) for [64\-bit] Windows, which disallows split / merged regions with +\fI\fBMEM_RELEASE\fR\fR\&. Although the same issues may present on 32\-bit platforms as well, retaining virtual memory for 32\-bit Linux and Windows is disabled by default due to the practical possibility of address space exhaustion\&. .RE .PP opt\&.dss (\fBconst char *\fR) r\- @@ -882,6 +896,13 @@ opt\&.narenas (\fBunsigned\fR) r\- Maximum number of arenas to use for automatic multiplexing of threads and arenas\&. The default is four times the number of CPUs, or one if there is a single CPU\&. .RE .PP +opt\&.oversize_threshold (\fBsize_t\fR) r\- +.RS 4 +The threshold in bytes of which requests are considered oversize\&. Allocation requests with greater sizes are fulfilled from a dedicated arena (automatically managed, however not within +narenas), in order to reduce fragmentation by not mixing huge allocations with small ones\&. In addition, the decay API guarantees on the extents greater than the specified threshold may be overridden\&. Note that requests with arena index specified via +\fBMALLOCX_ARENA\fR, or threads associated with explicit arenas will not be considered\&. The default threshold is 8MiB\&. Values not within large size classes disables this feature\&. +.RE +.PP opt\&.percpu_arena (\fBconst char *\fR) r\- .RS 4 Per CPU arena mode\&. Use the @@ -893,14 +914,14 @@ setting uses one arena per physical CPU, which means t \(lqdisabled\(rq\&. .RE .PP -opt\&.background_thread (\fBconst bool\fR) r\- +opt\&.background_thread (\fBbool\fR) r\- .RS 4 Internal background worker threads enabled/disabled\&. Because of potential circular dependencies, enabling background thread using this option may cause crash or deadlock during initialization\&. For a reliable way to use this feature, see background_thread for dynamic control options and details\&. This option is disabled by default\&. .RE .PP -opt\&.max_background_threads (\fBconst size_t\fR) r\- +opt\&.max_background_threads (\fBsize_t\fR) r\- .RS 4 Maximum number of background threads that will be created if background_thread @@ -917,7 +938,9 @@ and arena\&.<i>\&.dirty_decay_ms for related dynamic control options\&. See opt\&.muzzy_decay_ms -for a description of muzzy pages\&. +for a description of muzzy pages\&.for a description of muzzy pages\&. Note that when the +oversize_threshold +feature is enabled, the arenas reserved for oversize requests may have its own default decay settings\&. .RE .PP opt\&.muzzy_decay_ms (\fBssize_t\fR) r\- @@ -1293,7 +1316,7 @@ arena\&.<i>\&.extent_hooks (\fBextent_hooks_t *\fR) rw .RS 4 Get or set the extent management hook functions for arena <i>\&. The functions must be capable of operating on all extant extents associated with arena <i>, usually by passing unknown extents to the replaced functions\&. In practice, it is feasible to control allocation for arenas explicitly created via arenas\&.create -such that all extents originate from an application\-supplied extent allocator (by specifying the custom extent hook functions during arena creation), but the automatically created arenas will have already created extents prior to the application having an opportunity to take over extent allocation\&. +such that all extents originate from an application\-supplied extent allocator (by specifying the custom extent hook functions during arena creation)\&. However, the API guarantees for the automatically created arenas may be relaxed \-\- hooks set there may be called in a "best effort" fashion; in addition there may be extents created prior to the application having an opportunity to take over extent allocation\&. .sp .if n \{\ .RS 4 @@ -1876,6 +1899,11 @@ stats\&.retained for details\&. .RE .PP +stats\&.arenas\&.<i>\&.extent_avail (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Number of allocated (but unused) extent structs in this arena\&. +.RE +.PP stats\&.arenas\&.<i>\&.base (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Number of bytes dedicated to bootstrap\-sensitive allocator metadata structures\&. @@ -1956,6 +1984,16 @@ stats\&.arenas\&.<i>\&.small\&.nrequests (\fBuint64_t\ Cumulative number of allocation requests satisfied by all bin size classes\&. .RE .PP +stats\&.arenas\&.<i>\&.small\&.nfills (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Cumulative number of tcache fills by all small size classes\&. +.RE +.PP +stats\&.arenas\&.<i>\&.small\&.nflushes (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Cumulative number of tcache flushes by all small size classes\&. +.RE +.PP stats\&.arenas\&.<i>\&.large\&.allocated (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Number of bytes currently allocated by large objects\&. @@ -1980,6 +2018,16 @@ stats\&.arenas\&.<i>\&.large\&.nrequests (\fBuint64_t\ Cumulative number of allocation requests satisfied by all large size classes\&. .RE .PP +stats\&.arenas\&.<i>\&.large\&.nfills (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Cumulative number of tcache fills by all large size classes\&. +.RE +.PP +stats\&.arenas\&.<i>\&.large\&.nflushes (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Cumulative number of tcache flushes by all large size classes\&. +.RE +.PP stats\&.arenas\&.<i>\&.bins\&.<j>\&.nmalloc (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Cumulative number of times a bin region of the corresponding size class was allocated from the arena, whether to fill the relevant tcache if @@ -2029,6 +2077,11 @@ stats\&.arenas\&.<i>\&.bins\&.<j>\&.curslabs (\fBsize_ Current number of slabs\&. .RE .PP +stats\&.arenas\&.<i>\&.bins\&.<j>\&.nonfull_slabs (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Current number of nonfull slabs\&. +.RE +.PP stats\&.arenas\&.<i>\&.bins\&.<j>\&.mutex\&.{counter} (\fBcounter specific type\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Statistics on @@ -2037,6 +2090,16 @@ mutex (arena bin scope; bin operation related)\&. {counter} is one of the counters in mutex profiling counters\&. +.RE +.PP +stats\&.arenas\&.<i>\&.extents\&.<j>\&.n{extent_type} (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Number of extents of the given type in this arena in the bucket corresponding to page size index <j>\&. The extent type is one of dirty, muzzy, or retained\&. +.RE +.PP +stats\&.arenas\&.<i>\&.extents\&.<j>\&.{extent_type}_bytes (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] +.RS 4 +Sum of the bytes managed by extents of the given type in this arena in the bucket corresponding to page size index <j>\&. The extent type is one of dirty, muzzy, or retained\&. .RE .PP stats\&.arenas\&.<i>\&.lextents\&.<j>\&.nmalloc (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] Modified: head/contrib/jemalloc/include/jemalloc/internal/arena_externs.h ============================================================================== --- head/contrib/jemalloc/include/jemalloc/internal/arena_externs.h Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/include/jemalloc/internal/arena_externs.h Mon Nov 11 03:27:14 2019 (r354605) @@ -3,8 +3,8 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/extent_dss.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" extern ssize_t opt_dirty_decay_ms; @@ -16,13 +16,17 @@ extern const char *percpu_arena_mode_names[]; extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS]; extern malloc_mutex_t arenas_lock; +extern size_t opt_oversize_threshold; +extern size_t oversize_threshold; + void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - bin_stats_t *bstats, arena_stats_large_t *lstats); + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats); void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); #ifdef JEMALLOC_JET @@ -56,16 +60,17 @@ void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, szind_t ind, bool zero); void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache); -void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize); +void arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize); void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); -void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, - extent_t *extent, void *ptr); +void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr); bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); + size_t extra, bool zero, size_t *newsize); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache); + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_dirty_decay_ms_default_get(void); @@ -79,7 +84,12 @@ void arena_nthreads_inc(arena_t *arena, bool internal) void arena_nthreads_dec(arena_t *arena, bool internal); size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -void arena_boot(void); +bool arena_init_huge(void); +bool arena_is_huge(unsigned arena_ind); +arena_t *arena_choose_huge(tsd_t *tsd); +bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard); +void arena_boot(sc_data_t *sc_data); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); Modified: head/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h ============================================================================== --- head/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h Mon Nov 11 01:39:06 2019 (r354604) +++ head/contrib/jemalloc/include/jemalloc/internal/arena_inlines_b.h Mon Nov 11 03:27:14 2019 (r354605) @@ -4,10 +4,36 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" +JEMALLOC_ALWAYS_INLINE bool +arena_has_default_hooks(arena_t *arena) { + return (extent_hooks_get(arena) == &extent_hooks_default); +} + +JEMALLOC_ALWAYS_INLINE arena_t * +arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { + if (arena != NULL) { + return arena; + } + + /* + * For huge allocations, use the dedicated huge arena if both are true: + * 1) is using auto arena selection (i.e. arena == NULL), and 2) the + * thread is not assigned to a manual arena. + */ + if (unlikely(size >= oversize_threshold)) { + arena_t *tsd_arena = tsd_arena_get(tsd); + if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { + return arena_choose_huge(tsd); + } + } + + return arena_choose(tsd, NULL); +} + JEMALLOC_ALWAYS_INLINE prof_tctx_t * arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { cassert(config_prof); @@ -28,7 +54,7 @@ arena_prof_tctx_get(tsdn_t *tsdn, const void *ptr, all } JEMALLOC_ALWAYS_INLINE void -arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNUSED size_t usize, +arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -47,7 +73,7 @@ arena_prof_tctx_set(tsdn_t *tsdn, const void *ptr, UNU } static inline void -arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, UNUSED prof_tctx_t *tctx) { +arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { cassert(config_prof); assert(ptr != NULL); @@ -57,7 +83,33 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, U large_prof_tctx_reset(tsdn, extent); } +JEMALLOC_ALWAYS_INLINE nstime_t +arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, + alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + /* + * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're + * sure we have a sampled allocation. + */ + assert(!extent_slab_get(extent)); + return large_prof_alloc_time_get(extent); +} + JEMALLOC_ALWAYS_INLINE void +arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + assert(!extent_slab_get(extent)); + large_prof_alloc_time_set(extent, t); +} + +JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { tsd_t *tsd; ticker_t *decay_ticker; @@ -83,14 +135,33 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { arena_decay_ticks(tsdn, arena, 1); } +/* Purge a single extent to retained / unmapped directly. */ +JEMALLOC_ALWAYS_INLINE void +arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks, + extent_t *extent) { + size_t extent_size = extent_size_get(extent); + extent_dalloc_wrapper(tsdn, arena, + r_extent_hooks, extent); + if (config_stats) { + /* Update stats accordingly. */ + arena_stats_lock(tsdn, &arena->stats); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->nmadvise, 1); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); + arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, + extent_size); + arena_stats_unlock(tsdn, &arena->stats); + } +} + JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); - assert(size != 0); if (likely(tcache != NULL)) { - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); } @@ -119,7 +190,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) { szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } @@ -152,12 +223,22 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { /* Only slab members should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } static inline void +arena_dalloc_large_no_tcache(tsdn_t *tsdn, void *ptr, szind_t szind) { + if (config_prof && unlikely(szind < SC_NBINS)) { + arena_dalloc_promoted(tsdn, ptr, NULL, true); + } else { + extent_t *extent = iealloc(tsdn, ptr); + large_dalloc(tsdn, extent); + } +} + +static inline void arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { assert(ptr != NULL); @@ -173,7 +254,7 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -181,6 +262,21 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { /* Small allocation. */ arena_dalloc_small(tsdn, ptr); } else { + arena_dalloc_large_no_tcache(tsdn, ptr, szind); + } +} + +JEMALLOC_ALWAYS_INLINE void +arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind, + bool slow_path) { + if (szind < nhbins) { + if (config_prof && unlikely(szind < SC_NBINS)) { + arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); + } else { + tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, szind, + slow_path); + } + } else { extent_t *extent = iealloc(tsdn, ptr); large_dalloc(tsdn, extent); } @@ -203,7 +299,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache if (alloc_ctx != NULL) { szind = alloc_ctx->szind; slab = alloc_ctx->slab; - assert(szind != NSIZES); + assert(szind != SC_NSIZES); } else { rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, @@ -215,7 +311,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -224,25 +320,14 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); } else { - if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { - arena_dalloc_promoted(tsdn, ptr, tcache, - slow_path); - } else { - tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr, - szind, slow_path); - } - } else { - extent_t *extent = iealloc(tsdn, ptr); - large_dalloc(tsdn, extent); - } + arena_dalloc_large(tsdn, ptr, tcache, szind, slow_path); } } static inline void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); szind_t szind; bool slab; @@ -252,7 +337,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_ * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if ((config_prof && opt_prof) || config_debug) { @@ -264,7 +349,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_ (uintptr_t)ptr, true, &szind, &slab); assert(szind == sz_size2index(size)); - assert((config_prof && opt_prof) || slab == (szind < NBINS)); + assert((config_prof && opt_prof) || slab == (szind < SC_NBINS)); if (config_debug) { extent_t *extent = rtree_extent_read(tsdn, @@ -278,8 +363,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_ /* Small allocation. */ arena_dalloc_small(tsdn, ptr); } else { - extent_t *extent = iealloc(tsdn, ptr); - large_dalloc(tsdn, extent); + arena_dalloc_large_no_tcache(tsdn, ptr, szind); } } @@ -288,7 +372,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tc alloc_ctx_t *alloc_ctx, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); if (unlikely(tcache == NULL)) { arena_sdalloc_no_tcache(tsdn, ptr, size); @@ -297,7 +381,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tc szind_t szind; bool slab; - UNUSED alloc_ctx_t local_ctx; + alloc_ctx_t local_ctx; if (config_prof && opt_prof) { if (alloc_ctx == NULL) { /* Uncommon case and should be a static check. */ @@ -318,7 +402,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tc * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if (config_debug) { @@ -336,18 +420,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tc tcache_dalloc_small(tsdn_tsd(tsdn), tcache, ptr, szind, slow_path); } else { - if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { - arena_dalloc_promoted(tsdn, ptr, tcache, - slow_path); - } else { - tcache_dalloc_large(tsdn_tsd(tsdn), - tcache, ptr, szind, slow_path); - } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201911110327.xAB3REbS098393>