Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 24 Oct 2019 20:00:28 +0000 (UTC)
From:      Alexander Motin <mav@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org
Subject:   svn commit: r354041 - in stable/12/sys: amd64/amd64 dev/acpica vm
Message-ID:  <201910242000.x9OK0STL031601@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: mav
Date: Thu Oct 24 20:00:28 2019
New Revision: 354041
URL: https://svnweb.freebsd.org/changeset/base/354041

Log:
  MFC r351200 (by jeff), 351454,351456,351494-351495 (by kib):
  Allocate all per-cpu datastructures in domain correct memory.

Modified:
  stable/12/sys/amd64/amd64/mp_machdep.c
  stable/12/sys/dev/acpica/acpi_pxm.c
  stable/12/sys/dev/acpica/acpivar.h
  stable/12/sys/vm/vm_phys.c
  stable/12/sys/vm/vm_phys.h
Directory Properties:
  stable/12/   (props changed)

Modified: stable/12/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/12/sys/amd64/amd64/mp_machdep.c	Thu Oct 24 19:57:18 2019	(r354040)
+++ stable/12/sys/amd64/amd64/mp_machdep.c	Thu Oct 24 20:00:28 2019	(r354041)
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/cpuset.h>
+#include <sys/domainset.h>
 #ifdef GPROF 
 #include <sys/gmon.h>
 #endif
@@ -59,6 +60,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 #include <vm/vm_kern.h>
 #include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_phys.h>
 
 #include <x86/apicreg.h>
 #include <machine/clock.h>
@@ -75,6 +78,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <x86/init.h>
 
+#include <contrib/dev/acpica/include/acpi.h>
+#include <dev/acpica/acpivar.h>
+
 #define WARMBOOT_TARGET		0
 #define WARMBOOT_OFF		(KERNBASE + 0x0467)
 #define WARMBOOT_SEG		(KERNBASE + 0x0469)
@@ -385,6 +391,27 @@ init_secondary(void)
  * local functions and data
  */
 
+#ifdef NUMA
+static void
+mp_realloc_pcpu(int cpuid, int domain)
+{
+	vm_page_t m;
+	vm_offset_t oa, na;
+
+	oa = (vm_offset_t)&__pcpu[cpuid];
+	if (_vm_phys_domain(pmap_kextract(oa)) == domain)
+		return;
+	m = vm_page_alloc_domain(NULL, 0, domain,
+	    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
+	if (m == NULL)
+		return;
+	na = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
+	pagecopy((void *)oa, (void *)na);
+	pmap_enter(kernel_pmap, oa, m, VM_PROT_READ | VM_PROT_WRITE, 0, 0);
+	/* XXX old pcpu page leaked. */
+}
+#endif
+
 /*
  * start each AP in our list
  */
@@ -393,7 +420,7 @@ native_start_all_aps(void)
 {
 	u_int64_t *pt4, *pt3, *pt2;
 	u_int32_t mpbioswarmvec;
-	int apic_id, cpu, i;
+	int apic_id, cpu, domain, i;
 	u_char mpbiosreason;
 
 	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
@@ -432,21 +459,39 @@ native_start_all_aps(void)
 	outb(CMOS_REG, BIOS_RESET);
 	outb(CMOS_DATA, BIOS_WARM);	/* 'warm-start' */
 
+	/* Relocate pcpu areas to the correct domain. */
+#ifdef NUMA
+	if (vm_ndomains > 1)
+		for (cpu = 1; cpu < mp_ncpus; cpu++) {
+			apic_id = cpu_apic_ids[cpu];
+			domain = acpi_pxm_get_cpu_locality(apic_id);
+			mp_realloc_pcpu(cpu, domain);
+		}
+#endif
+
 	/* start each AP */
+	domain = 0;
 	for (cpu = 1; cpu < mp_ncpus; cpu++) {
 		apic_id = cpu_apic_ids[cpu];
-
+#ifdef NUMA
+		if (vm_ndomains > 1)
+			domain = acpi_pxm_get_cpu_locality(apic_id);
+#endif
 		/* allocate and set up an idle stack data page */
 		bootstacks[cpu] = (void *)kmem_malloc(kstack_pages * PAGE_SIZE,
 		    M_WAITOK | M_ZERO);
 		doublefault_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK |
 		    M_ZERO);
 		mce_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
-		nmi_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
-		dbg_stack = (char *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
-		dpcpu = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
+		nmi_stack = (char *)kmem_malloc_domainset(
+		    DOMAINSET_PREF(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+		dbg_stack = (char *)kmem_malloc_domainset(
+		    DOMAINSET_PREF(domain), PAGE_SIZE, M_WAITOK | M_ZERO);
+		dpcpu = (void *)kmem_malloc_domainset(DOMAINSET_PREF(domain),
+		    DPCPU_SIZE, M_WAITOK | M_ZERO);
 
-		bootSTK = (char *)bootstacks[cpu] + kstack_pages * PAGE_SIZE - 8;
+		bootSTK = (char *)bootstacks[cpu] +
+		    kstack_pages * PAGE_SIZE - 8;
 		bootAP = cpu;
 
 		/* attempt to start the Application Processor */

Modified: stable/12/sys/dev/acpica/acpi_pxm.c
==============================================================================
--- stable/12/sys/dev/acpica/acpi_pxm.c	Thu Oct 24 19:57:18 2019	(r354040)
+++ stable/12/sys/dev/acpica/acpi_pxm.c	Thu Oct 24 20:00:28 2019	(r354041)
@@ -653,6 +653,17 @@ acpi_pxm_set_cpu_locality(void)
 	}
 }
 
+int
+acpi_pxm_get_cpu_locality(int apic_id)
+{
+	struct cpu_info *cpu;
+
+	cpu = cpu_find(apic_id);
+	if (cpu == NULL)
+		panic("SRAT: CPU with ID %u is not known", apic_id);
+	return (cpu->domain);
+}
+
 /*
  * Free data structures allocated during acpi_pxm_init.
  */

Modified: stable/12/sys/dev/acpica/acpivar.h
==============================================================================
--- stable/12/sys/dev/acpica/acpivar.h	Thu Oct 24 19:57:18 2019	(r354040)
+++ stable/12/sys/dev/acpica/acpivar.h	Thu Oct 24 20:00:28 2019	(r354041)
@@ -528,6 +528,7 @@ int		acpi_pxm_init(int ncpus, vm_paddr_t maxphys);
 void		acpi_pxm_parse_tables(void);
 void		acpi_pxm_set_mem_locality(void);
 void		acpi_pxm_set_cpu_locality(void);
+int		acpi_pxm_get_cpu_locality(int apic_id);
 void		acpi_pxm_free(void);
 
 /*

Modified: stable/12/sys/vm/vm_phys.c
==============================================================================
--- stable/12/sys/vm/vm_phys.c	Thu Oct 24 19:57:18 2019	(r354040)
+++ stable/12/sys/vm/vm_phys.c	Thu Oct 24 20:00:28 2019	(r354041)
@@ -624,6 +624,26 @@ vm_phys_register_domains(int ndomains, struct mem_affi
 #endif
 }
 
+int
+_vm_phys_domain(vm_paddr_t pa)
+{
+#ifdef NUMA
+	int i;
+
+	if (vm_ndomains == 1 || mem_affinity == NULL)
+		return (0);
+
+	/*
+	 * Check for any memory that overlaps.
+	 */
+	for (i = 0; mem_affinity[i].end != 0; i++)
+		if (mem_affinity[i].start <= pa &&
+		    mem_affinity[i].end >= pa)
+			return (mem_affinity[i].domain);
+#endif
+	return (0);
+}
+
 /*
  * Split a contiguous, power of two-sized set of physical pages.
  *

Modified: stable/12/sys/vm/vm_phys.h
==============================================================================
--- stable/12/sys/vm/vm_phys.h	Thu Oct 24 19:57:18 2019	(r354040)
+++ stable/12/sys/vm/vm_phys.h	Thu Oct 24 20:00:28 2019	(r354041)
@@ -122,6 +122,7 @@ vm_phys_domain(vm_page_t m)
 	return (0);
 #endif
 }
+int _vm_phys_domain(vm_paddr_t pa);
 
 #endif	/* _KERNEL */
 #endif	/* !_VM_PHYS_H_ */



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201910242000.x9OK0STL031601>