Supposed fix for some SandyBridge mobile CPUs hang on AP startup when

x2APIC mode is detected and enabled.  Current theory is that switching
the APIC mode while an IPI is in flight might be the issue.

Postpone switching to x2APIC mode until we are guaranteed that all
starting IPIs are already send and aknowledged.  Use aps_ready signal
as an indication that the BSP is done with us.

Tested by:	adrian
Sponsored by:	The FreeBSD Foundation
MFC after:	2 months

sys/amd64/amd64/mp_machdep.c

@@ -705,19 +705,21 @@ init_secondary(void)
 	wrmsr(MSR_STAR, msr);
 	wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D);
 
-	/*
-	 * On real hardware, switch to x2apic mode if possible.
-	 * Disable local APIC until BSP directed APs to run.
-	 */
-	lapic_xapic_mode();
-
 	/* signal our startup to the BSP. */
 	mp_naps++;
 
 	/* Spin until the BSP releases the AP's. */
 	while (!aps_ready)
 		ia32_pause();
 
+	/*
+	 * On real hardware, switch to x2apic mode if possible.  Do it
+	 * after aps_ready was signalled, to avoid manipulating the
+	 * mode while BSP might still want to send some IPI to us
+	 * (second startup IPI is ignored on modern hardware etc).
+	 */
+	lapic_xapic_mode();
+
 	/* Initialize the PAT MSR. */
 	pmap_init_pat();
 

sys/i386/i386/mp_machdep.c

@@ -719,12 +719,6 @@ init_secondary(void)
 	load_cr0(cr0);
 	CHECK_WRITE(0x38, 5);
 
-	/*
-	 * On real hardware, switch to x2apic mode if possible.
-	 * Disable local APIC until BSP directed APs to run.
-	 */
-	lapic_xapic_mode();
-
 	/* signal our startup to the BSP. */
 	mp_naps++;
 	CHECK_WRITE(0x39, 6);
@@ -742,6 +736,14 @@ init_secondary(void)
 	lidt(&r_idt);
 #endif
 
+	/*
+	 * On real hardware, switch to x2apic mode if possible.  Do it
+	 * after aps_ready was signalled, to avoid manipulating the
+	 * mode while BSP might still want to send some IPI to us
+	 * (second startup IPI is ignored on modern hardware etc).
+	 */
+	lapic_xapic_mode();
+
 	/* Initialize the PAT MSR if present. */
 	pmap_init_pat();