Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 19 May 2020 16:04:27 +0000 (UTC)
From:      Andrew Turner <andrew@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r361259 - in head/sys/arm64: arm64 include
Message-ID:  <202005191604.04JG4RvR002450@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: andrew
Date: Tue May 19 16:04:27 2020
New Revision: 361259
URL: https://svnweb.freebsd.org/changeset/base/361259

Log:
  Stop performing a full icache sync when the DIC and IDC flags are set
  
  The DIC and IDC bits in the CTR_EL0 register signal to the kernel when it
  can relax the instruction cache synchronisation operations. The IDC bit
  means we can relax cleaning the data cache to the point of unification
  while the DIC bit means we don't need to invalidate the instruction cache
  for data coherence. In both cases an appropriate barrier is still needed.
  
  For now only implement the case where both bits are set, as is the case
  on the Neoverse-N1 as used in the Amazon AWS Graviton 2 CPU. Note that
  this behaviour is a optional on the N1 so we may later need to implement
  only one or the other bit being set.
  
  There is a tunable to disable each flag on boot.
  
  Testing on a 4 core Graviton 2 instance found a significant improvement
  in sys and real time when running "make buildkernel -j4", with no
  significant difference in user time.
  
  Reviewed by:	markj
  Sponsored by:	Innovate UK
  Differential Revision:	https://reviews.freebsd.org/D24853

Modified:
  head/sys/arm64/arm64/cpufunc_asm.S
  head/sys/arm64/arm64/identcpu.c
  head/sys/arm64/include/cpufunc.h

Modified: head/sys/arm64/arm64/cpufunc_asm.S
==============================================================================
--- head/sys/arm64/arm64/cpufunc_asm.S	Tue May 19 15:27:20 2020	(r361258)
+++ head/sys/arm64/arm64/cpufunc_asm.S	Tue May 19 16:04:27 2020	(r361259)
@@ -133,9 +133,20 @@ ENTRY(arm64_dcache_inv_range)
 END(arm64_dcache_inv_range)
 
 /*
- * void arm64_icache_sync_range(vm_offset_t, vm_size_t)
+ * void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t)
+ * When the CTR_EL0.IDC bit is set cleaning to PoU becomes a dsb.
+ * When the CTR_EL0.DIC bit is set icache invalidation becomes an isb.
  */
-ENTRY(arm64_icache_sync_range)
+ENTRY(arm64_dic_idc_icache_sync_range)
+	dsb	ishst
+	isb
+	ret
+END(arm64_dic_idc_icache_sync_range)
+
+/*
+ * void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t)
+ */
+ENTRY(arm64_aliasing_icache_sync_range)
 	/*
 	 * XXX Temporary solution - I-cache flush should be range based for
 	 * PIPT cache or IALLUIS for VIVT or VIPT caches
@@ -146,7 +157,7 @@ ENTRY(arm64_icache_sync_range)
 	dsb	ish
 	isb
 	ret
-END(arm64_icache_sync_range)
+END(arm64_aliasing_icache_sync_range)
 
 /*
  * int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t)

Modified: head/sys/arm64/arm64/identcpu.c
==============================================================================
--- head/sys/arm64/arm64/identcpu.c	Tue May 19 15:27:20 2020	(r361258)
+++ head/sys/arm64/arm64/identcpu.c	Tue May 19 16:04:27 2020	(r361259)
@@ -56,6 +56,24 @@ char machine[] = "arm64";
 extern int adaptive_machine_arch;
 #endif
 
+static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+    "Cache management tuning");
+
+static int allow_dic = 1;
+SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
+    "Allow optimizations based on the DIC cache bit");
+
+static int allow_idc = 1;
+SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
+    "Allow optimizations based on the IDC cache bit");
+
+/*
+ * The default implementation of I-cache sync assumes we have an
+ * aliasing cache until we know otherwise.
+ */
+void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t) =
+    &arm64_aliasing_icache_sync_range;
+
 static int
 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
 {
@@ -977,6 +995,7 @@ identify_cpu_sysinit(void *dummy __unused)
 {
 	int cpu;
 	u_long hwcap;
+	bool dic, idc;
 
 	/* Create a user visible cpu description with safe values */
 	memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
@@ -985,6 +1004,8 @@ identify_cpu_sysinit(void *dummy __unused)
 	    ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | ID_AA64PFR0_EL0_64;
 	user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
 
+	dic = (allow_dic != 0);
+	idc = (allow_idc != 0);
 	CPU_FOREACH(cpu) {
 		print_cpu_features(cpu);
 		hwcap = parse_cpu_features_hwcap(cpu);
@@ -993,6 +1014,17 @@ identify_cpu_sysinit(void *dummy __unused)
 		else
 			elf_hwcap &= hwcap;
 		update_user_regs(cpu);
+
+		if (CTR_DIC_VAL(cpu_desc[cpu].ctr) == 0)
+			dic = false;
+		if (CTR_IDC_VAL(cpu_desc[cpu].ctr) == 0)
+			idc = false;
+	}
+
+	if (dic && idc) {
+		arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
+		if (bootverbose)
+			printf("Enabling DIC & IDC ICache sync\n");
 	}
 
 	if ((elf_hwcap & HWCAP_ATOMICS) != 0) {

Modified: head/sys/arm64/include/cpufunc.h
==============================================================================
--- head/sys/arm64/include/cpufunc.h	Tue May 19 15:27:20 2020	(r361258)
+++ head/sys/arm64/include/cpufunc.h	Tue May 19 16:04:27 2020	(r361259)
@@ -216,12 +216,15 @@ extern int64_t dczva_line_size;
 #define	cpu_dcache_inv_range(a, s)	arm64_dcache_inv_range((a), (s))
 #define	cpu_dcache_wb_range(a, s)	arm64_dcache_wb_range((a), (s))
 
+extern void (*arm64_icache_sync_range)(vm_offset_t, vm_size_t);
+
 #define	cpu_icache_sync_range(a, s)	arm64_icache_sync_range((a), (s))
 #define cpu_icache_sync_range_checked(a, s) arm64_icache_sync_range_checked((a), (s))
 
 void arm64_nullop(void);
 void arm64_tlb_flushID(void);
-void arm64_icache_sync_range(vm_offset_t, vm_size_t);
+void arm64_dic_idc_icache_sync_range(vm_offset_t, vm_size_t);
+void arm64_aliasing_icache_sync_range(vm_offset_t, vm_size_t);
 int arm64_icache_sync_range_checked(vm_offset_t, vm_size_t);
 void arm64_dcache_wbinv_range(vm_offset_t, vm_size_t);
 void arm64_dcache_inv_range(vm_offset_t, vm_size_t);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?202005191604.04JG4RvR002450>