summaryrefslogtreecommitdiff
path: root/arch/x86/lib/delay.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/lib/delay.c')
-rw-r--r--arch/x86/lib/delay.c60
1 files changed, 51 insertions, 9 deletions
diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
index 39d6a3db0..e912b2f6d 100644
--- a/arch/x86/lib/delay.c
+++ b/arch/x86/lib/delay.c
@@ -20,6 +20,7 @@
#include <asm/processor.h>
#include <asm/delay.h>
#include <asm/timer.h>
+#include <asm/mwait.h>
#ifdef CONFIG_SMP
# include <asm/smp.h>
@@ -49,16 +50,14 @@ static void delay_loop(unsigned long loops)
/* TSC based delay: */
static void delay_tsc(unsigned long __loops)
{
- u32 bclock, now, loops = __loops;
+ u64 bclock, now, loops = __loops;
int cpu;
preempt_disable();
cpu = smp_processor_id();
- rdtsc_barrier();
- rdtscl(bclock);
+ bclock = rdtsc_ordered();
for (;;) {
- rdtsc_barrier();
- rdtscl(now);
+ now = rdtsc_ordered();
if ((now - bclock) >= loops)
break;
@@ -79,14 +78,51 @@ static void delay_tsc(unsigned long __loops)
if (unlikely(cpu != smp_processor_id())) {
loops -= (now - bclock);
cpu = smp_processor_id();
- rdtsc_barrier();
- rdtscl(bclock);
+ bclock = rdtsc_ordered();
}
}
preempt_enable();
}
/*
+ * On some AMD platforms, MWAITX has a configurable 32-bit timer, that
+ * counts with TSC frequency. The input value is the loop of the
+ * counter, it will exit when the timer expires.
+ */
+static void delay_mwaitx(unsigned long __loops)
+{
+ u64 start, end, delay, loops = __loops;
+
+ start = rdtsc_ordered();
+
+ for (;;) {
+ delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
+
+ /*
+ * Use cpu_tss as a cacheline-aligned, seldomly
+ * accessed per-cpu variable as the monitor target.
+ */
+ __monitorx(this_cpu_ptr(&cpu_tss), 0, 0);
+
+ /*
+ * AMD, like Intel, supports the EAX hint and EAX=0xf
+ * means, do not enter any deep C-state and we use it
+ * here in delay() to minimize wakeup latency.
+ */
+ __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
+
+ end = rdtsc_ordered();
+
+ if (loops <= end - start)
+ break;
+
+ loops -= end - start;
+
+ start = end;
+ }
+}
+
+/*
* Since we calibrate only once at boot, this
* function should be set once at boot and not changed
*/
@@ -94,13 +130,19 @@ static void (*delay_fn)(unsigned long) = delay_loop;
void use_tsc_delay(void)
{
- delay_fn = delay_tsc;
+ if (delay_fn == delay_loop)
+ delay_fn = delay_tsc;
+}
+
+void use_mwaitx_delay(void)
+{
+ delay_fn = delay_mwaitx;
}
int read_current_timer(unsigned long *timer_val)
{
if (delay_fn == delay_tsc) {
- rdtscll(*timer_val);
+ *timer_val = rdtsc();
return 0;
}
return -1;