Date: Sat, 16 Feb 2019 16:29:52 -0800 From: Mark Millard <marklmi@yahoo.com> To: Justin Hibbits <chmeeedalf@gmail.com> Cc: FreeBSD PowerPC ML <freebsd-ppc@freebsd.org> Subject: Cleaned -up evidence about the PowerMac G5 multiprocessor boot hang ups with the modern VM_MAX_KERNEL_ADDRESS value Message-ID: <E7826D8D-4BA3-4A73-B006-9E7FBD1376B1@yahoo.com>
next in thread | raw e-mail | index | archive | help
I eliminated a bunch of reports that gave information
that was the same for hang-ups and successful boots.
So this is starting over with less material to go
through.
For an unsuccessful boot:
(Values like 0x10, 0x25, 0x51 are use as recorded labels for points in =
the code.)
Adding CPU 0, hwref=3Dcd38, awake=3D1
Waking up CPU 3 (dev=3Dc480)
After reset 4&0 for CPU 3, hwref=3Dc480, awake=3D0, *(unsigned =
long*)0xc0000000000000f0=3D0x25
After attempted wait for awake CPU 3, hwref=3Dc480, awake=3D0, =
*(unsigned long*)0xc0000000000000f0=3D0x25
Waking up CPU 2 (dev=3Dc768)
So neither pc_awake nor *0xc0000000000000f0 have machdep_ap_bootstrap
based values as seen by CPU 0 (from trying to start CPU 3).
For a successful boot:
Adding CPU 0, hwref=3Dcd38, awake=3D1
Waking up CPU 3 (dev=3Dc480)
After reset 4&0 for CPU 3, hwref=3Dc480, awake=3D0, *(unsigned =
long*)0xc0000000000000f0=3D0x25
After attempted wait for awake CPU 3, hwref=3Dc480, awake=3D1, =
*(unsigned long*)0xc0000000000000f0=3D0x51
Adding CPU 3, hwref=3Dc480, awake=3D1
Waking up CPU 2 (dev=3Dc768)
After reset 4&0 for CPU 2, hwref=3Dc768, awake=3D0, *(unsigned =
long*)0xc0000000000000f0=3D0x51
After attempted wait for awake CPU 2, hwref=3Dc768, awake=3D1, =
*(unsigned long*)0xc0000000000000f0=3D0x51
Adding CPU 2, hwref=3Dc768, awake=3D1
Waking up CPU 1 (dev=3Dca50)
After reset 4&0 for CPU 1, hwref=3Dca50, awake=3D0, *(unsigned =
long*)0xc0000000000000f0=3D0x51
After attempted wait for awake CPU 1, hwref=3Dca50, awake=3D1, =
*(unsigned long*)0xc0000000000000f0=3D0x51
Adding CPU 1, hwref=3Dca50, awake=3D1
SMP: AP CPU #3 launched
SMP: AP CPU #1 launched
SMP: AP CPU #2 launched
Trying to mount root from ufs:/dev/ufs/FBSDG5L2rootfs [rw,noatime]...
GEOM: new disk ada1
So here waiting on cpu 0 does see pc_awake become 1 and does see
0xc0000000000000f0 updated by machdep_ap_bootstrap for CPU , for
example.
Note the awake=3D0 (i.e.., pc_awake) examples for each cpu even when
there is already a 0x51 label for CPU 2 and CPU 3. (The code is
shown later.)
This is based on the labeling updates to:
("labels" stored at 0xc0000000000000f0)
cpudep_ap_early_bootstrap (before-return labeled with value : 0x10)
moea64_cpu_bootstrap_native (labeled with values: 0x21, 0x22, 0x23, =
0x24, 0x25 at various points)
pmap_cpu_bootstrap (before-return labeled with value : 0x20)
cpudep_ap_bootstrap (before-return labeled with value : 0x30)
cpudep_ap_setup (before-return labeled with value : 0x40)
machdep_ap_bootstrap (lableled with values: 0x5F, 0x51, and 0x50 =
at various points)
powermac_smp_start_cpu (reporting values after the resets for a CPU =
and after
the pa_awake loop)
I happen to have experimented with additional isyncs in
this variant of the code, not that they made much of
a difference.
The specifics are:
# svnlite diff /usr/src/sys/powerpc/aim/mp_cpudep.c =
/usr/src/sys/powerpc/powerpc/pmap_dispatch.c =
/usr/src/sys/powerpc/powerpc/mp_machdep.c =
/usr/src/sys/powerpc/powermac/platform_powermac.c =
/usr/src/sys/powerpc/aim/moea64_native.c | more
Index: /usr/src/sys/powerpc/aim/mp_cpudep.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- /usr/src/sys/powerpc/aim/mp_cpudep.c (revision 344018)
+++ /usr/src/sys/powerpc/aim/mp_cpudep.c (working copy)
@@ -105,6 +105,10 @@
=20
__asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu));
powerpc_sync();
+
+ *(unsigned long*)0xc0000000000000f0 =3D 0x10; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
}
=20
uintptr_t
@@ -124,6 +128,10 @@
pcpup->pc_curpcb =3D pcpup->pc_curthread->td_pcb;
sp =3D pcpup->pc_curpcb->pcb_sp;
=20
+ *(unsigned long*)0xc0000000000000f0 =3D 0x30; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
+
return (sp);
}
=20
@@ -416,5 +424,9 @@
"suboptimal.\n");
break;
}
+
+ *(unsigned long*)0xc0000000000000f0 =3D 0x40; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
}
=20
Index: /usr/src/sys/powerpc/aim/moea64_native.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- /usr/src/sys/powerpc/aim/moea64_native.c (revision 344018)
+++ /usr/src/sys/powerpc/aim/moea64_native.c (working copy)
@@ -384,6 +384,9 @@
=20
mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR);
=20
+ *(unsigned long*)0xc0000000000000f0 =3D 0x21; // HACK!!!
+ powerpc_sync(); // HACK!!!
+
/*
* Install kernel SLB entries
*/
@@ -393,6 +396,9 @@
__asm __volatile ("slbmfee %0,%1; slbie %0;" : =
"=3Dr"(seg0) :
"r"(0));
=20
+ *(unsigned long*)0xc0000000000000f0 =3D 0x22; // HACK!!!
+ powerpc_sync(); // HACK!!!
+
for (i =3D 0; i < n_slbs; i++) {
if (!(slb[i].slbe & SLBE_VALID))
continue;
@@ -400,6 +406,9 @@
__asm __volatile ("slbmte %0, %1" ::=20
"r"(slb[i].slbv), "r"(slb[i].slbe));=20
}
+
+ *(unsigned long*)0xc0000000000000f0 =3D 0x23; // HACK!!!
+ powerpc_sync(); // HACK!!!
#else
for (i =3D 0; i < 16; i++)
mtsrin(i << ADDR_SR_SHFT, =
kernel_pmap->pm_sr[i]);
@@ -412,7 +421,14 @@
__asm __volatile ("ptesync; mtsdr1 %0; isync"
:: "r"(((uintptr_t)moea64_pteg_table & ~DMAP_BASE_ADDRESS)
| (uintptr_t)(flsl(moea64_pteg_mask >> 11))));
+
+ *(unsigned long*)0xc0000000000000f0 =3D 0x24; // HACK!!!
+ powerpc_sync(); // HACK!!!
+
tlbia();
+
+ *(unsigned long*)0xc0000000000000f0 =3D 0x25; // HACK!!!
+ powerpc_sync(); // HACK!!!
}
Index: /usr/src/sys/powerpc/powerpc/pmap_dispatch.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- /usr/src/sys/powerpc/powerpc/pmap_dispatch.c (revision =
344018)
+++ /usr/src/sys/powerpc/powerpc/pmap_dispatch.c (working copy)
@@ -445,6 +445,10 @@
*/
=20
return (MMU_CPU_BOOTSTRAP(mmu_obj, ap));
+
+ *(unsigned long*)0xc0000000000000f0 =3D 0x20; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
}
=20
void *
Index: /usr/src/sys/powerpc/powerpc/mp_machdep.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- /usr/src/sys/powerpc/powerpc/mp_machdep.c (revision 344018)
+++ /usr/src/sys/powerpc/powerpc/mp_machdep.c (working copy)
@@ -73,12 +73,20 @@
void
machdep_ap_bootstrap(void)
{
+ *(unsigned long*)0xc0000000000000f0 =3D 0x5F; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
=20
PCPU_SET(awake, 1);
__asm __volatile("msync; isync");
=20
+ *(unsigned long*)0xc0000000000000f0 =3D 0x51; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
+
while (ap_letgo =3D=3D 0)
__asm __volatile("or 31,31,31");
+ isync(); // HACK!!!
__asm __volatile("or 6,6,6");
=20
/*
@@ -109,10 +117,15 @@
=20
while(smp_started =3D=3D 0)
;
+ isync(); // HACK!!!
=20
/* Start per-CPU event timers. */
cpu_initclocks_ap();
=20
+ *(unsigned long*)0xc0000000000000f0 =3D 0x50; // HACK!!!
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
+
/* Announce ourselves awake, and enter the scheduler */
sched_throw(NULL);
}
@@ -241,11 +254,13 @@
timeout =3D 2000; /* wait 2sec for the =
AP */
while (!pc->pc_awake && --timeout > 0)
DELAY(1000);
+ isync(); // HACK!!!
}
} else {
pc->pc_awake =3D 1;
}
if (pc->pc_awake) {
+ isync(); // HACK!!!
if (bootverbose)
printf("Adding CPU %d, hwref=3D%jx, =
awake=3D%x\n",
pc->pc_cpuid, =
(uintmax_t)pc->pc_hwref,
@@ -256,6 +271,8 @@
}
=20
ap_awake =3D 1;
+ isync(); // HACK!!!
+ powerpc_sync(); // HACK!!!
=20
/* Provide our current DEC and TB values for APs */
ap_timebase =3D mftb() + 10;
@@ -268,6 +285,7 @@
=20
while (ap_awake < smp_cpus)
;
+ isync(); // HACK!!!
=20
if (smp_cpus !=3D cpus || cpus !=3D mp_ncpus) {
printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs =
woken\n",
Index: /usr/src/sys/powerpc/powermac/platform_powermac.c
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D
--- /usr/src/sys/powerpc/powermac/platform_powermac.c (revision =
344018)
+++ /usr/src/sys/powerpc/powermac/platform_powermac.c (working copy)
@@ -382,10 +382,22 @@
(void)(*rstvec);
powerpc_sync();
=20
+ isync(); // HACK!!!
+ if (bootverbose) // HACK!!!
+ printf("After reset 4&0 for CPU %d, hwref=3D%jx, =
awake=3D%x, *(unsigned long*)0xc0000000000000f0=3D0x%jx\n",
+ pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+ pc->pc_awake,(uintmax_t)*(unsigned =
long*)0xc0000000000000f0);
+
timeout =3D 10000;
while (!pc->pc_awake && timeout--)
DELAY(100);
+ isync(); // HACK!!!
=20
+ if (bootverbose) // HACK!!!
+ printf("After attempted wait for awake CPU %d, =
hwref=3D%jx, awake=3D%x, *(unsigned long*)0xc0000000000000f0=3D0x%jx\n",
+ pc->pc_cpuid, (uintmax_t)pc->pc_hwref,
+ pc->pc_awake,(uintmax_t)*(unsigned =
long*)0xc0000000000000f0);
+
return ((pc->pc_awake) ? 0 : EBUSY);
#else
/* No SMP support */
=3D=3D=3D
Mark Millard
marklmi at yahoo.com
( dsl-only.net went
away in early 2018-Mar)
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?E7826D8D-4BA3-4A73-B006-9E7FBD1376B1>