From owner-svn-src-all@freebsd.org Wed Nov 27 21:00:44 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id A949A1B8C40; Wed, 27 Nov 2019 21:00:44 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 47NY8D43Rpz4btx; Wed, 27 Nov 2019 21:00:44 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 6E7ED1B421; Wed, 27 Nov 2019 21:00:44 +0000 (UTC) (envelope-from dougm@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xARL0ipm028969; Wed, 27 Nov 2019 21:00:44 GMT (envelope-from dougm@FreeBSD.org) Received: (from dougm@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xARL0iYu028968; Wed, 27 Nov 2019 21:00:44 GMT (envelope-from dougm@FreeBSD.org) Message-Id: <201911272100.xARL0iYu028968@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: dougm set sender to dougm@FreeBSD.org using -f From: Doug Moore Date: Wed, 27 Nov 2019 21:00:44 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r355147 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: dougm X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 355147 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 27 Nov 2019 21:00:44 -0000 Author: dougm Date: Wed Nov 27 21:00:44 2019 New Revision: 355147 URL: https://svnweb.freebsd.org/changeset/base/355147 Log: Inline some splay helper functions to improve performance on a micro-benchmark. Reviewed by: markj Tested by: pho Differential Revision: https://reviews.freebsd.org/D22544 Modified: head/sys/vm/vm_map.c Modified: head/sys/vm/vm_map.c ============================================================================== --- head/sys/vm/vm_map.c Wed Nov 27 20:33:53 2019 (r355146) +++ head/sys/vm/vm_map.c Wed Nov 27 21:00:44 2019 (r355147) @@ -1069,53 +1069,48 @@ vm_map_entry_pred(vm_map_entry_t entry) * the subsequent call to vm_map_splay_merge, rely on the start and end address * values in &map->header. */ -static vm_map_entry_t +static __always_inline vm_map_entry_t vm_map_splay_split(vm_map_t map, vm_offset_t addr, vm_size_t length, - vm_map_entry_t *out_llist, vm_map_entry_t *out_rlist) + vm_map_entry_t *llist, vm_map_entry_t *rlist) { - vm_map_entry_t llist, rlist, root, y; + vm_map_entry_t root, y; - llist = rlist = &map->header; + *llist = *rlist = &map->header; root = map->root; while (root != NULL && root->max_free >= length) { - KASSERT(llist->end <= root->start && root->end <= rlist->start, + KASSERT((*llist)->end <= root->start && + root->end <= (*rlist)->start, ("%s: root not within tree bounds", __func__)); if (addr < root->start) { - SPLAY_LEFT_STEP(root, y, rlist, + SPLAY_LEFT_STEP(root, y, *rlist, y->max_free >= length && addr < y->start); } else if (addr >= root->end) { - SPLAY_RIGHT_STEP(root, y, llist, + SPLAY_RIGHT_STEP(root, y, *llist, y->max_free >= length && addr >= y->end); } else break; } - *out_llist = llist; - *out_rlist = rlist; return (root); } -static void -vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *iolist) +static __always_inline void +vm_map_splay_findnext(vm_map_entry_t root, vm_map_entry_t *rlist) { - vm_map_entry_t rlist, y; + vm_map_entry_t hi, y; - root = root->right; - rlist = *iolist; - while (root != NULL) - SPLAY_LEFT_STEP(root, y, rlist, true); - *iolist = rlist; + hi = root->right; + while (hi != NULL) + SPLAY_LEFT_STEP(hi, y, *rlist, true); } -static void -vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *iolist) +static __always_inline void +vm_map_splay_findprev(vm_map_entry_t root, vm_map_entry_t *llist) { - vm_map_entry_t llist, y; + vm_map_entry_t lo, y; - root = root->left; - llist = *iolist; - while (root != NULL) - SPLAY_RIGHT_STEP(root, y, llist, true); - *iolist = llist; + lo = root->left; + while (lo != NULL) + SPLAY_RIGHT_STEP(lo, y, *llist, true); } static inline void @@ -1270,30 +1265,22 @@ vm_map_entry_unlink(vm_map_t map, vm_map_entry_t entry KASSERT(root != NULL, ("vm_map_entry_unlink: unlink object not mapped")); + vm_map_splay_findprev(root, &llist); vm_map_splay_findnext(root, &rlist); - switch (op) { - case UNLINK_MERGE_NEXT: + if (op == UNLINK_MERGE_NEXT) { rlist->start = root->start; rlist->offset = root->offset; - y = root->left; + } + if (llist != &map->header) { + root = llist; + llist = root->right; + root->right = NULL; + } else if (rlist != &map->header) { root = rlist; rlist = root->left; - root->left = y; - break; - case UNLINK_MERGE_NONE: - vm_map_splay_findprev(root, &llist); - if (llist != &map->header) { - root = llist; - llist = root->right; - root->right = NULL; - } else if (rlist != &map->header) { - root = rlist; - rlist = root->left; - root->left = NULL; - } else - root = NULL; - break; - } + root->left = NULL; + } else + root = NULL; y = entry->next; y->prev = entry->prev; y->prev->next = y; @@ -1322,8 +1309,7 @@ vm_map_entry_resize(vm_map_t map, vm_map_entry_t entry VM_MAP_ASSERT_LOCKED(map); root = vm_map_splay_split(map, entry->start, 0, &llist, &rlist); - KASSERT(root != NULL, - ("%s: resize object not mapped", __func__)); + KASSERT(root != NULL, ("%s: resize object not mapped", __func__)); vm_map_splay_findnext(root, &rlist); root->right = NULL; entry->end += grow_amount;