Skip site navigation (1)Skip section navigation (2) 
Date:      Sun, 12 May 2019 06:36:54 +0000 (UTC)
From:      Mateusz Guzik <mjg@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r347502 - in head/sys: amd64/amd64 amd64/include i386/i386 i386/include x86/include x86/x86
Message-ID:  <201905120636.x4C6asUq094481@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help 
Author: mjg
Date: Sun May 12 06:36:54 2019
New Revision: 347502
URL: https://svnweb.freebsd.org/changeset/base/347502

Log:
  x86: store pending bitmapped IPIs in per-cpu areas
  
  This gets rid of the global cpu_ipi_pending array.
  
  While replace cmpset with fcmpset in the delivery code and opportunistically
  check if given IPI is already pending.
  
  Sponsored by:	The FreeBSD Foundation

Modified:
  head/sys/amd64/amd64/mp_machdep.c
  head/sys/amd64/include/pcpu.h
  head/sys/i386/i386/mp_machdep.c
  head/sys/i386/include/pcpu.h
  head/sys/x86/include/x86_smp.h
  head/sys/x86/x86/mp_x86.c

Modified: head/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- head/sys/amd64/amd64/mp_machdep.c	Sun May 12 06:34:58 2019	(r347501)
+++ head/sys/amd64/amd64/mp_machdep.c	Sun May 12 06:36:54 2019	(r347502)
@@ -193,7 +193,6 @@ cpu_mp_start(void)
 	/* Initialize the logical ID to APIC ID table. */
 	for (i = 0; i < MAXCPU; i++) {
 		cpu_apic_ids[i] = -1;
-		cpu_ipi_pending[i] = 0;
 	}
 
 	/* Install an inter-CPU IPI for TLB invalidation */

Modified: head/sys/amd64/include/pcpu.h
==============================================================================
--- head/sys/amd64/include/pcpu.h	Sun May 12 06:34:58 2019	(r347501)
+++ head/sys/amd64/include/pcpu.h	Sun May 12 06:36:54 2019	(r347502)
@@ -84,7 +84,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
 	uint32_t pc_pcid_gen;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
 	uint32_t pc_ibpb_set;						\
-	char	__pad[3288]		/* pad to UMA_PCPU_ALLOC_SIZE */
+	u_int 	pc_ipi_bitmap;						\
+	char	__pad[3284]		/* pad to UMA_PCPU_ALLOC_SIZE */
 
 #define	PC_DBREG_CMD_NONE	0
 #define	PC_DBREG_CMD_LOAD	1

Modified: head/sys/i386/i386/mp_machdep.c
==============================================================================
--- head/sys/i386/i386/mp_machdep.c	Sun May 12 06:34:58 2019	(r347501)
+++ head/sys/i386/i386/mp_machdep.c	Sun May 12 06:36:54 2019	(r347502)
@@ -153,7 +153,6 @@ cpu_mp_start(void)
 	/* Initialize the logical ID to APIC ID table. */
 	for (i = 0; i < MAXCPU; i++) {
 		cpu_apic_ids[i] = -1;
-		cpu_ipi_pending[i] = 0;
 	}
 
 	/* Install an inter-CPU IPI for TLB invalidation */

Modified: head/sys/i386/include/pcpu.h
==============================================================================
--- head/sys/i386/include/pcpu.h	Sun May 12 06:34:58 2019	(r347501)
+++ head/sys/i386/include/pcpu.h	Sun May 12 06:36:54 2019	(r347502)
@@ -87,7 +87,8 @@ _Static_assert(sizeof(struct monitorbuf) == 128, "2x c
 	caddr_t pc_pmap_eh_ptep;						\
 	uint32_t pc_smp_tlb_done;	/* TLB op acknowledgement */	\
 	uint32_t pc_ibpb_set;						\
-	char	__pad[3610]
+	u_int	pc_ipi_bitmap;						\
+	char	__pad[3606]
 
 #ifdef _KERNEL
 

Modified: head/sys/x86/include/x86_smp.h
==============================================================================
--- head/sys/x86/include/x86_smp.h	Sun May 12 06:34:58 2019	(r347501)
+++ head/sys/x86/include/x86_smp.h	Sun May 12 06:36:54 2019	(r347502)
@@ -34,7 +34,6 @@ extern char *bootSTK;
 extern void *bootstacks[];
 extern unsigned int boot_address;
 extern unsigned int bootMP_size;
-extern volatile u_int cpu_ipi_pending[];
 extern volatile int aps_ready;
 extern struct mtx ap_boot_mtx;
 extern int cpu_logical;

Modified: head/sys/x86/x86/mp_x86.c
==============================================================================
--- head/sys/x86/x86/mp_x86.c	Sun May 12 06:34:58 2019	(r347501)
+++ head/sys/x86/x86/mp_x86.c	Sun May 12 06:36:54 2019	(r347502)
@@ -137,9 +137,6 @@ _Static_assert(MAXCPU <= MAX_APIC_ID,
 _Static_assert(xAPIC_MAX_APIC_ID <= MAX_APIC_ID,
     "xAPIC_MAX_APIC_ID cannot be larger that MAX_APIC_ID");
 
-/* Holds pending bitmap based IPIs per CPU */
-volatile u_int cpu_ipi_pending[MAXCPU];
-
 static void	release_aps(void *dummy);
 static void	cpustop_handler_post(u_int cpu);
 
@@ -1224,19 +1221,24 @@ ipi_startup(int apic_id, int vector)
 void
 ipi_send_cpu(int cpu, u_int ipi)
 {
-	u_int bitmap, old_pending, new_pending;
+	u_int bitmap, old, new;
+	u_int *cpu_bitmap;
 
 	KASSERT(cpu_apic_ids[cpu] != -1, ("IPI to non-existent CPU %d", cpu));
 
 	if (IPI_IS_BITMAPED(ipi)) {
 		bitmap = 1 << ipi;
 		ipi = IPI_BITMAP_VECTOR;
-		do {
-			old_pending = cpu_ipi_pending[cpu];
-			new_pending = old_pending | bitmap;
-		} while  (!atomic_cmpset_int(&cpu_ipi_pending[cpu],
-		    old_pending, new_pending));	
-		if (old_pending)
+		cpu_bitmap = &cpuid_to_pcpu[cpu]->pc_ipi_bitmap;
+		old = *cpu_bitmap;
+		for (;;) {
+			if ((old & bitmap) == bitmap)
+				break;
+			new = old | bitmap;
+			if (atomic_fcmpset_int(cpu_bitmap, &old, new))
+				break;
+		}
+		if (old)
 			return;
 	}
 	lapic_ipi_vectored(ipi, cpu_apic_ids[cpu]);
@@ -1255,7 +1257,7 @@ ipi_bitmap_handler(struct trapframe frame)
 	td->td_intr_nesting_level++;
 	oldframe = td->td_intr_frame;
 	td->td_intr_frame = &frame;
-	ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
+	ipi_bitmap = atomic_readandclear_int(&cpuid_to_pcpu[cpu]->pc_ipi_bitmap);
 	if (ipi_bitmap & (1 << IPI_PREEMPT)) {
 #ifdef COUNT_IPIS
 		(*ipi_preempt_counts[cpu])++;

Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201905120636.x4C6asUq094481>