From owner-svn-src-head@freebsd.org Wed Feb 12 11:11:23 2020 Return-Path: Delivered-To: svn-src-head@mailman.nyi.freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2610:1c1:1:606c::19:1]) by mailman.nyi.freebsd.org (Postfix) with ESMTP id 5594B234878; Wed, 12 Feb 2020 11:11:23 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from mxrelay.nyi.freebsd.org (mxrelay.nyi.freebsd.org [IPv6:2610:1c1:1:606c::19:3]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) server-signature RSA-PSS (4096 bits) client-signature RSA-PSS (4096 bits) client-digest SHA256) (Client CN "mxrelay.nyi.freebsd.org", Issuer "Let's Encrypt Authority X3" (verified OK)) by mx1.freebsd.org (Postfix) with ESMTPS id 48HcQg1gdgz4FHq; Wed, 12 Feb 2020 11:11:23 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org (repo.freebsd.org [IPv6:2610:1c1:1:6068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mxrelay.nyi.freebsd.org (Postfix) with ESMTPS id 1C45C201E9; Wed, 12 Feb 2020 11:11:23 +0000 (UTC) (envelope-from mjg@FreeBSD.org) Received: from repo.freebsd.org ([127.0.1.37]) by repo.freebsd.org (8.15.2/8.15.2) with ESMTP id 01CBBMsE083864; Wed, 12 Feb 2020 11:11:22 GMT (envelope-from mjg@FreeBSD.org) Received: (from mjg@localhost) by repo.freebsd.org (8.15.2/8.15.2/Submit) id 01CBBMxr083861; Wed, 12 Feb 2020 11:11:22 GMT (envelope-from mjg@FreeBSD.org) Message-Id: <202002121111.01CBBMxr083861@repo.freebsd.org> X-Authentication-Warning: repo.freebsd.org: mjg set sender to mjg@FreeBSD.org using -f From: Mateusz Guzik Date: Wed, 12 Feb 2020 11:11:22 +0000 (UTC) To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r357804 - in head/sys: kern riscv/include sys vm X-SVN-Group: head X-SVN-Commit-Author: mjg X-SVN-Commit-Paths: in head/sys: kern riscv/include sys vm X-SVN-Commit-Revision: 357804 X-SVN-Commit-Repository: base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-head@freebsd.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: SVN commit messages for the src tree for head/-current List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 12 Feb 2020 11:11:23 -0000 Author: mjg Date: Wed Feb 12 11:11:22 2020 New Revision: 357804 URL: https://svnweb.freebsd.org/changeset/base/357804 Log: Store offset into zpcpu allocations in the per-cpu area. This shorten zpcpu_get and allows more optimizations. Reviewed by: jeff Differential Revision: https://reviews.freebsd.org/D23570 Modified: head/sys/kern/subr_pcpu.c head/sys/riscv/include/pcpu.h head/sys/sys/pcpu.h head/sys/vm/uma_core.c Modified: head/sys/kern/subr_pcpu.c ============================================================================== --- head/sys/kern/subr_pcpu.c Wed Feb 12 11:10:10 2020 (r357803) +++ head/sys/kern/subr_pcpu.c Wed Feb 12 11:11:22 2020 (r357804) @@ -95,6 +95,7 @@ pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) cpu_pcpu_init(pcpu, cpuid, size); pcpu->pc_rm_queue.rmq_next = &pcpu->pc_rm_queue; pcpu->pc_rm_queue.rmq_prev = &pcpu->pc_rm_queue; + pcpu->pc_zpcpu_offset = zpcpu_offset_cpu(cpuid); } void Modified: head/sys/riscv/include/pcpu.h ============================================================================== --- head/sys/riscv/include/pcpu.h Wed Feb 12 11:10:10 2020 (r357803) +++ head/sys/riscv/include/pcpu.h Wed Feb 12 11:11:22 2020 (r357804) @@ -48,7 +48,7 @@ struct pmap *pc_curpmap; /* Currently active pmap */ \ uint32_t pc_pending_ipis; /* IPIs pending to this CPU */ \ uint32_t pc_hart; /* Hart ID */ \ - char __pad[57] + char __pad[49] #ifdef _KERNEL Modified: head/sys/sys/pcpu.h ============================================================================== --- head/sys/sys/pcpu.h Wed Feb 12 11:10:10 2020 (r357803) +++ head/sys/sys/pcpu.h Wed Feb 12 11:11:22 2020 (r357804) @@ -194,6 +194,7 @@ struct pcpu { struct rm_queue pc_rm_queue; /* rmlock list of trackers */ uintptr_t pc_dynamic; /* Dynamic per-cpu data area */ uint64_t pc_early_dummy_counter; /* Startup time counter(9) */ + uintptr_t pc_zpcpu_offset; /* Offset into zpcpu allocs */ /* * Keep MD fields last, so that CPU-specific variations on a @@ -227,14 +228,28 @@ extern struct pcpu *cpuid_to_pcpu[]; #endif #define curproc (curthread->td_proc) +#ifndef zpcpu_offset_cpu +#define zpcpu_offset_cpu(cpu) (UMA_PCPU_ALLOC_SIZE * cpu) +#endif +#ifndef zpcpu_offset +#define zpcpu_offset() (PCPU_GET(zpcpu_offset)) +#endif + +#ifndef zpcpu_base_to_offset +#define zpcpu_base_to_offset(base) (base) +#endif +#ifndef zpcpu_offset_to_base +#define zpcpu_offset_to_base(base) (base) +#endif + /* Accessor to elements allocated via UMA_ZONE_PCPU zone. */ #define zpcpu_get(base) ({ \ - __typeof(base) _ptr = (void *)((char *)(base) + UMA_PCPU_ALLOC_SIZE * curcpu); \ + __typeof(base) _ptr = (void *)((char *)(base) + zpcpu_offset()); \ _ptr; \ }) #define zpcpu_get_cpu(base, cpu) ({ \ - __typeof(base) _ptr = (void *)((char *)(base) + UMA_PCPU_ALLOC_SIZE * cpu); \ + __typeof(base) _ptr = (void *)((char *)(base) + zpcpu_offset_cpu(cpu)); \ _ptr; \ }) Modified: head/sys/vm/uma_core.c ============================================================================== --- head/sys/vm/uma_core.c Wed Feb 12 11:10:10 2020 (r357803) +++ head/sys/vm/uma_core.c Wed Feb 12 11:11:22 2020 (r357804) @@ -2944,34 +2944,39 @@ uma_zwait(uma_zone_t zone) void * uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) { - void *item; + void *item, *pcpu_item; #ifdef SMP int i; MPASS(zone->uz_flags & UMA_ZONE_PCPU); #endif item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); - if (item != NULL && (flags & M_ZERO)) { + if (item == NULL) + return (NULL); + pcpu_item = zpcpu_base_to_offset(item); + if (flags & M_ZERO) { #ifdef SMP for (i = 0; i <= mp_maxid; i++) - bzero(zpcpu_get_cpu(item, i), zone->uz_size); + bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); #else bzero(item, zone->uz_size); #endif } - return (item); + return (pcpu_item); } /* * A stub while both regular and pcpu cases are identical. */ void -uma_zfree_pcpu_arg(uma_zone_t zone, void *item, void *udata) +uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) { + void *item; #ifdef SMP MPASS(zone->uz_flags & UMA_ZONE_PCPU); #endif + item = zpcpu_offset_to_base(pcpu_item); uma_zfree_arg(zone, item, udata); }