From owner-svn-src-all@freebsd.org Wed Nov 20 00:31:15 2019 Return-Path: Delivered-To: svn-src-all@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 4D02B1C2E02; Wed, 20 Nov 2019 00:31:15 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 47HkBq1LGyz47lG; Wed, 20 Nov 2019 00:31:15 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 1219E1AAC0; Wed, 20 Nov 2019 00:31:15 +0000 (UTC) (envelope-from jeff@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id xAK0VEcU010304; Wed, 20 Nov 2019 00:31:14 GMT (envelope-from jeff@FreeBSD.org) Received: (from jeff@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id xAK0VEeH010302; Wed, 20 Nov 2019 00:31:14 GMT (envelope-from jeff@FreeBSD.org) Message-Id: <201911200031.xAK0VEeH010302@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: jeff set sender to jeff@FreeBSD.org using -f From: Jeff Roberson Date: Wed, 20 Nov 2019 00:31:14 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r354871 - head/sys/vm X-SVN-Group: head X-SVN-Commit-Author: jeff X-SVN-Commit-Paths: head/sys/vm X-SVN-Commit-Revision: 354871 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-all@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "SVN commit messages for the entire src tree \(except for " user" and " projects" \)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 20 Nov 2019 00:31:15 -0000 Author: jeff Date: Wed Nov 20 00:31:14 2019 New Revision: 354871 URL: https://svnweb.freebsd.org/changeset/base/354871 Log: Only keep anonymous objects on shadow lists. This eliminates locking of globally visible objects when they are part of a backing chain. Reviewed by: kib, markj Differential Revision: https://reviews.freebsd.org/D22423 Modified: head/sys/vm/vm_object.c head/sys/vm/vm_object.h Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Tue Nov 19 23:30:09 2019 (r354870) +++ head/sys/vm/vm_object.c Wed Nov 20 00:31:14 2019 (r354871) @@ -118,6 +118,7 @@ static boolean_t vm_object_page_remove_write(vm_page_t boolean_t *allclean); static void vm_object_qcollapse(vm_object_t object); static void vm_object_vndeallocate(vm_object_t object); +static void vm_object_backing_remove(vm_object_t object); /* * Virtual memory objects maintain the actual data @@ -651,11 +652,7 @@ doterm: if (temp != NULL) { KASSERT((object->flags & OBJ_TMPFS_NODE) == 0, ("shadowed tmpfs v_object 2 %p", object)); - VM_OBJECT_WLOCK(temp); - LIST_REMOVE(object, shadow_list); - temp->shadow_count--; - VM_OBJECT_WUNLOCK(temp); - object->backing_object = NULL; + vm_object_backing_remove(object); } /* * Don't double-terminate, we could be in a termination @@ -695,6 +692,70 @@ vm_object_destroy(vm_object_t object) uma_zfree(obj_zone, object); } +static void +vm_object_backing_remove_locked(vm_object_t object) +{ + vm_object_t backing_object; + + backing_object = object->backing_object; + VM_OBJECT_ASSERT_WLOCKED(object); + VM_OBJECT_ASSERT_WLOCKED(backing_object); + + if ((object->flags & OBJ_SHADOWLIST) != 0) { + LIST_REMOVE(object, shadow_list); + backing_object->shadow_count--; + object->flags &= ~OBJ_SHADOWLIST; + } + object->backing_object = NULL; +} + +static void +vm_object_backing_remove(vm_object_t object) +{ + vm_object_t backing_object; + + VM_OBJECT_ASSERT_WLOCKED(object); + + if ((object->flags & OBJ_SHADOWLIST) != 0) { + backing_object = object->backing_object; + VM_OBJECT_WLOCK(backing_object); + vm_object_backing_remove_locked(object); + VM_OBJECT_WUNLOCK(backing_object); + } else + object->backing_object = NULL; +} + +static void +vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object) +{ + + VM_OBJECT_ASSERT_WLOCKED(object); + + if ((backing_object->flags & OBJ_ANON) != 0) { + VM_OBJECT_ASSERT_WLOCKED(backing_object); + LIST_INSERT_HEAD(&backing_object->shadow_head, object, + shadow_list); + backing_object->shadow_count++; + object->flags |= OBJ_SHADOWLIST; + } + object->backing_object = backing_object; +} + +static void +vm_object_backing_insert(vm_object_t object, vm_object_t backing_object) +{ + + VM_OBJECT_ASSERT_WLOCKED(object); + + if ((backing_object->flags & OBJ_ANON) != 0) { + VM_OBJECT_WLOCK(backing_object); + vm_object_backing_insert_locked(object, backing_object); + VM_OBJECT_WUNLOCK(backing_object); + } else + object->backing_object = backing_object; +} + + /* * vm_object_terminate_pages removes any remaining pageable pages * from the object and resets the object to an empty state. @@ -1250,6 +1311,12 @@ vm_object_shadow( result = vm_object_allocate_anon(atop(length)); /* + * Store the offset into the source object, and fix up the offset into + * the new object. + */ + result->backing_object_offset = *offset; + + /* * The new object shadows the source object, adding a reference to it. * Our caller changes his reference to point to the new object, * removing a reference to the source object. Net result: no change @@ -1259,26 +1326,18 @@ vm_object_shadow( * in order to maintain page coloring consistency in the combined * shadowed object. */ - result->backing_object = source; - /* - * Store the offset into the source object, and fix up the offset into - * the new object. - */ - result->backing_object_offset = *offset; if (source != NULL) { - VM_OBJECT_WLOCK(source); + VM_OBJECT_WLOCK(result); + vm_object_backing_insert(result, source); result->domain = source->domain; - LIST_INSERT_HEAD(&source->shadow_head, result, shadow_list); - source->shadow_count++; #if VM_NRESERVLEVEL > 0 result->flags |= source->flags & OBJ_COLORED; result->pg_color = (source->pg_color + OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER - 1)) - 1); #endif - VM_OBJECT_WUNLOCK(source); + VM_OBJECT_WUNLOCK(result); } - /* * Return the new things */ @@ -1326,24 +1385,26 @@ vm_object_split(vm_map_entry_t entry) new_object->domain = orig_object->domain; source = orig_object->backing_object; if (source != NULL) { - VM_OBJECT_WLOCK(source); - if ((source->flags & OBJ_DEAD) != 0) { + if ((source->flags & (OBJ_ANON | OBJ_DEAD)) != 0) { + VM_OBJECT_WLOCK(source); + if ((source->flags & OBJ_DEAD) != 0) { + VM_OBJECT_WUNLOCK(source); + VM_OBJECT_WUNLOCK(orig_object); + VM_OBJECT_WUNLOCK(new_object); + vm_object_deallocate(new_object); + VM_OBJECT_WLOCK(orig_object); + return; + } + vm_object_backing_insert_locked(new_object, source); + vm_object_reference_locked(source); /* for new_object */ + vm_object_clear_flag(source, OBJ_ONEMAPPING); VM_OBJECT_WUNLOCK(source); - VM_OBJECT_WUNLOCK(orig_object); - VM_OBJECT_WUNLOCK(new_object); - vm_object_deallocate(new_object); - VM_OBJECT_WLOCK(orig_object); - return; + } else { + vm_object_backing_insert(new_object, source); + vm_object_reference(source); } - LIST_INSERT_HEAD(&source->shadow_head, - new_object, shadow_list); - source->shadow_count++; - vm_object_reference_locked(source); /* for new_object */ - vm_object_clear_flag(source, OBJ_ONEMAPPING); - VM_OBJECT_WUNLOCK(source); new_object->backing_object_offset = orig_object->backing_object_offset + entry->offset; - new_object->backing_object = source; } if (orig_object->cred != NULL) { new_object->cred = orig_object->cred; @@ -1756,20 +1817,15 @@ vm_object_collapse(vm_object_t object) * backing_object->backing_object moves from within * backing_object to within object. */ - LIST_REMOVE(object, shadow_list); - backing_object->shadow_count--; - if (backing_object->backing_object) { - VM_OBJECT_WLOCK(backing_object->backing_object); - LIST_REMOVE(backing_object, shadow_list); - LIST_INSERT_HEAD( - &backing_object->backing_object->shadow_head, - object, shadow_list); - /* - * The shadow_count has not changed. - */ - VM_OBJECT_WUNLOCK(backing_object->backing_object); + vm_object_backing_remove_locked(object); + new_backing_object = backing_object->backing_object; + if (new_backing_object != NULL) { + VM_OBJECT_WLOCK(new_backing_object); + vm_object_backing_remove_locked(backing_object); + vm_object_backing_insert_locked(object, + new_backing_object); + VM_OBJECT_WUNLOCK(new_backing_object); } - object->backing_object = backing_object->backing_object; object->backing_object_offset += backing_object->backing_object_offset; @@ -1807,20 +1863,13 @@ vm_object_collapse(vm_object_t object) * chain. Deallocating backing_object will not remove * it, since its reference count is at least 2. */ - LIST_REMOVE(object, shadow_list); - backing_object->shadow_count--; + vm_object_backing_remove_locked(object); new_backing_object = backing_object->backing_object; - if ((object->backing_object = new_backing_object) != NULL) { - VM_OBJECT_WLOCK(new_backing_object); - LIST_INSERT_HEAD( - &new_backing_object->shadow_head, - object, - shadow_list - ); - new_backing_object->shadow_count++; - vm_object_reference_locked(new_backing_object); - VM_OBJECT_WUNLOCK(new_backing_object); + if (new_backing_object != NULL) { + vm_object_backing_insert(object, + new_backing_object); + vm_object_reference(new_backing_object); object->backing_object_offset += backing_object->backing_object_offset; } Modified: head/sys/vm/vm_object.h ============================================================================== --- head/sys/vm/vm_object.h Tue Nov 19 23:30:09 2019 (r354870) +++ head/sys/vm/vm_object.h Wed Nov 20 00:31:14 2019 (r354871) @@ -192,6 +192,7 @@ struct vm_object { #define OBJ_TMPFS_NODE 0x0200 /* object belongs to tmpfs VREG node */ #define OBJ_COLORED 0x1000 /* pg_color is defined */ #define OBJ_ONEMAPPING 0x2000 /* One USE (a single, non-forked) mapping flag */ +#define OBJ_SHADOWLIST 0x4000 /* Object is on the shadow list. */ #define OBJ_TMPFS 0x8000 /* has tmpfs vnode allocated */ /*