From owner-svn-src-all@freebsd.org Sat Nov 9 17:08:28 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 9143B1AEEC9; Sat, 9 Nov 2019 17:08:28 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 479NrX2pBgz3xVV; Sat, 9 Nov 2019 17:08:28 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 4487719D10; Sat, 9 Nov 2019 17:08:28 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xA9H8SV4080214; Sat, 9 Nov 2019 17:08:28 GMT (envelope-from dougm@FreeBSD.org) Received: (from dougm@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xA9H8R5M080213; Sat, 9 Nov 2019 17:08:27 GMT (envelope-from dougm@FreeBSD.org) Message-Id: <201911091708.xA9H8R5M080213@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: dougm set sender to dougm@FreeBSD.org using -f From: Doug Moore Date: Sat, 9 Nov 2019 17:08:27 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r354570 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: dougm X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 354570 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sat, 09 Nov 2019 17:08:28 -0000 Author: dougm Date: Sat Nov 9 17:08:27 2019 New Revision: 354570 URL: https://svnweb.freebsd.org/changeset/base/354570 Log: For vm_map, #defining DIAGNOSTIC to turn on full assertion-based consistency checking slows performance dramatically. This change reduces the number of assertions checked by completely walking the vm_map tree only when the write-lock is released, and only then if the number of modifications to the tree since the last walk exceeds the number of tree nodes. Reviewed by: alc, kib Tested by: pho Differential Revision: https://reviews.freebsd.org/D22163 Modified: head/sys/vm/vm_map.c head/sys/vm/vm_map.h Modified: head/sys/vm/vm_map.c ============================================================================== --- head/sys/vm/vm_map.c Sat Nov 9 11:55:01 2019 (r354569) +++ head/sys/vm/vm_map.c Sat Nov 9 17:08:27 2019 (r354570) @@ -596,10 +596,54 @@ vm_map_process_deferred(void) } } +#ifdef INVARIANTS +static void +_vm_map_assert_locked(vm_map_t map, const char *file, int line) +{ + + if (map->system_map) + mtx_assert_(&map->system_mtx, MA_OWNED, file, line); + else + sx_assert_(&map->lock, SA_XLOCKED, file, line); +} + +#define VM_MAP_ASSERT_LOCKED(map) \ + _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE) + +enum { VMMAP_CHECK_NONE, VMMAP_CHECK_UNLOCK, VMMAP_CHECK_ALL }; +#ifdef DIAGNOSTIC +static int enable_vmmap_check = VMMAP_CHECK_UNLOCK; +#else +static int enable_vmmap_check = VMMAP_CHECK_NONE; +#endif +SYSCTL_INT(_debug, OID_AUTO, vmmap_check, CTLFLAG_RWTUN, + &enable_vmmap_check, 0, "Enable vm map consistency checking"); + +static void _vm_map_assert_consistent(vm_map_t map, int check); + +#define VM_MAP_ASSERT_CONSISTENT(map) \ + _vm_map_assert_consistent(map, VMMAP_CHECK_ALL) +#ifdef DIAGNOSTIC +#define VM_MAP_UNLOCK_CONSISTENT(map) do { \ + if (map->nupdates > map->nentries) { \ + _vm_map_assert_consistent(map, VMMAP_CHECK_UNLOCK); \ + map->nupdates = 0; \ + } \ +} while (0) +#else +#define VM_MAP_UNLOCK_CONSISTENT(map) +#endif +#else +#define VM_MAP_ASSERT_LOCKED(map) +#define VM_MAP_ASSERT_CONSISTENT(map) +#define VM_MAP_UNLOCK_CONSISTENT(map) +#endif /* INVARIANTS */ + void _vm_map_unlock(vm_map_t map, const char *file, int line) { + VM_MAP_UNLOCK_CONSISTENT(map); if (map->system_map) mtx_unlock_flags_(&map->system_mtx, 0, file, line); else { @@ -697,8 +741,10 @@ _vm_map_lock_downgrade(vm_map_t map, const char *file, if (map->system_map) { mtx_assert_(&map->system_mtx, MA_OWNED, file, line); - } else + } else { + VM_MAP_UNLOCK_CONSISTENT(map); sx_downgrade_(&map->lock, file, line); + } } /* @@ -717,37 +763,6 @@ vm_map_locked(vm_map_t map) return (sx_xlocked(&map->lock)); } -#ifdef INVARIANTS -static void -_vm_map_assert_locked(vm_map_t map, const char *file, int line) -{ - - if (map->system_map) - mtx_assert_(&map->system_mtx, MA_OWNED, file, line); - else - sx_assert_(&map->lock, SA_XLOCKED, file, line); -} - -#define VM_MAP_ASSERT_LOCKED(map) \ - _vm_map_assert_locked(map, LOCK_FILE, LOCK_LINE) - -#ifdef DIAGNOSTIC -static int enable_vmmap_check = 1; -#else -static int enable_vmmap_check = 0; -#endif -SYSCTL_INT(_debug, OID_AUTO, vmmap_check, CTLFLAG_RWTUN, - &enable_vmmap_check, 0, "Enable vm map consistency checking"); - -static void _vm_map_assert_consistent(vm_map_t map); - -#define VM_MAP_ASSERT_CONSISTENT(map) \ - _vm_map_assert_consistent(map) -#else -#define VM_MAP_ASSERT_LOCKED(map) -#define VM_MAP_ASSERT_CONSISTENT(map) -#endif /* INVARIANTS */ - /* * _vm_map_unlock_and_wait: * @@ -766,6 +781,7 @@ int _vm_map_unlock_and_wait(vm_map_t map, int timo, const char *file, int line) { + VM_MAP_UNLOCK_CONSISTENT(map); mtx_lock(&map_sleep_mtx); if (map->system_map) mtx_unlock_flags_(&map->system_mtx, 0, file, line); @@ -874,6 +890,9 @@ _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t mi map->timestamp = 0; map->busy = 0; map->anon_loc = 0; +#ifdef DIAGNOSTIC + map->nupdates = 0; +#endif } void @@ -1132,6 +1151,9 @@ vm_map_splay_merge(vm_map_t map, vm_map_entry_t root, } root->max_free = MAX(max_free_left, max_free_right); map->root = root; +#ifdef DIAGNOSTIC + ++map->nupdates; +#endif } /* @@ -1330,8 +1352,10 @@ vm_map_lookup_entry( * on a temporary upgrade. */ cur = vm_map_splay(map, address); - if (!locked) + if (!locked) { + VM_MAP_UNLOCK_CONSISTENT(map); sx_downgrade(&map->lock); + } /* * If "address" is contained within a map entry, the new root @@ -4786,12 +4810,12 @@ vm_map_pmap_KBI(vm_map_t map) #ifdef INVARIANTS static void -_vm_map_assert_consistent(vm_map_t map) +_vm_map_assert_consistent(vm_map_t map, int check) { vm_map_entry_t entry, prev; vm_size_t max_left, max_right; - if (!enable_vmmap_check) + if (enable_vmmap_check != check) return; prev = &map->header; Modified: head/sys/vm/vm_map.h ============================================================================== --- head/sys/vm/vm_map.h Sat Nov 9 11:55:01 2019 (r354569) +++ head/sys/vm/vm_map.h Sat Nov 9 17:08:27 2019 (r354570) @@ -207,6 +207,9 @@ struct vm_map { pmap_t pmap; /* (c) Physical map */ vm_offset_t anon_loc; int busy; +#ifdef DIAGNOSTIC + int nupdates; +#endif }; /*