[PATCH 4/6] x86: introduce rdtsc_barrier()

Alok Kataria akataria at vmware.com
Tue Jun 3 23:33:47 UTC 2008


OriginalAuthor: Andi Kleen <ak at suse.de>
Commit: fde1b3fa947c2512e3715962ebb1d3a6a9b9bb7d

rdtsc_barrier() is a new barrier primitive that stops RDTSC speculation
to avoid races with timer interrupts on other CPUs.

It expands either to LFENCE (for Intel CPUs) or MFENCE (for
AMD CPUs) which stops RDTSC on all currently known microarchitectures
that implement SSE. On CPUs without SSE there is generally no RDTSC
speculation.

Signed-off-by: Alok N Kataria <akataria at vmware.com>
---
 include/asm-x86/system_32.h |   14 ++++++++++++++
 include/asm-x86/system_64.h |   14 ++++++++++++++
 2 files changed, 28 insertions(+), 0 deletions(-)

diff --git a/include/asm-x86/system_32.h b/include/asm-x86/system_32.h
index ef84688..ea3e1d6 100644
--- a/include/asm-x86/system_32.h
+++ b/include/asm-x86/system_32.h
@@ -5,6 +5,7 @@
 #include <asm/segment.h>
 #include <asm/cpufeature.h>
 #include <asm/cmpxchg.h>
+#include <asm/nops.h>
 
 #ifdef __KERNEL__
 #define AT_VECTOR_SIZE_ARCH 2 /* entries in ARCH_DLINFO */
@@ -299,6 +300,19 @@ static inline unsigned long get_limit(unsigned long segment)
 #define set_mb(var, value) do { var = value; barrier(); } while (0)
 #endif
 
+/*
+ * Stop RDTSC speculation. This is needed when you need to use RDTSC
+ * (or get_cycles or vread that possibly accesses the TSC) in a defined
+ * code region.
+ *
+ * (Could use an alternative three way for this if there was one.)
+ */
+static inline void rdtsc_barrier(void)
+{
+	alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
+	alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
+}
+
 #include <linux/irqflags.h>
 
 /*
diff --git a/include/asm-x86/system_64.h b/include/asm-x86/system_64.h
index 6e9e484..a7618ad 100644
--- a/include/asm-x86/system_64.h
+++ b/include/asm-x86/system_64.h
@@ -4,6 +4,7 @@
 #include <linux/kernel.h>
 #include <asm/segment.h>
 #include <asm/cmpxchg.h>
+#include <asm/nops.h>
 
 #ifdef __KERNEL__
 
@@ -173,6 +174,19 @@ static inline void clflush(volatile void *__p)
 #define read_barrier_depends()	do {} while(0)
 #define set_mb(var, value) do { (void) xchg(&var, value); } while (0)
 
+/*
+ * Stop RDTSC speculation. This is needed when you need to use RDTSC
+ * (or get_cycles or vread that possibly accesses the TSC) in a defined
+ * code region.
+ *
+ * (Could use an alternative three way for this if there was one.)
+ */
+static inline void rdtsc_barrier(void)
+{
+	alternative(ASM_NOP3, "mfence", X86_FEATURE_MFENCE_RDTSC);
+	alternative(ASM_NOP3, "lfence", X86_FEATURE_LFENCE_RDTSC);
+}
+
 #define warn_if_not_ulong(x) do { unsigned long foo; (void) (&(x) == &foo); } while (0)
 
 #include <linux/irqflags.h>
-- 
1.5.4







More information about the kernel-team mailing list