| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Thu, 21 May 2015 10:55:44 +0300
From: Adrian Hunter <adrian.hunter@...el.com>
To: Thomas Gleixner <tglx@...utronix.de>
Cc: linux-kernel@...r.kernel.org,
Andy Lutomirski <luto@...capital.net>,
Linus Torvalds <torvalds@...ux-foundation.org>,
Andi Kleen <ak@...ux.intel.com>, x86@...nel.org,
"H. Peter Anvin" <hpa@...or.com>
Subject: [PATCH RFC] x86, tsc: Allow for high latency in quick_pit_calibrate()
If it takes longer than 12us to read the PIT counter lsb/msb,
then the error margin will never fall below 500ppm within 50ms,
and Fast TSC calibration will always fail.
This patch detects when that will happen and switches to using
a slightly different algorithm that takes advantage of the PIT's
latch comand.
Signed-off-by: Adrian Hunter <adrian.hunter@...el.com>
---
arch/x86/kernel/tsc.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 132 insertions(+), 1 deletion(-)
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 5054497..0035682 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -553,6 +553,40 @@ static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *de
}
/*
+ * High latency version of pit_expect_msb(). Instead of using the read latency
+ * as the error margin, latch the counter and use that latency. The counter
+ * latches at the current value which means there is an addition error margin
+ * of 1 timer tick which is not accounted for here.
+ */
+static inline int pit_expect_msb_hl(unsigned char val, u64 *tscp,
+ unsigned long *deltap, u16 *cnt)
+{
+ u64 tsc0, tsc1;
+ int count;
+ u8 lsb, msb;
+
+ for (count = 0; count < 50000; count++) {
+ tsc0 = get_cycles();
+ /* Latch counter 2 */
+ outb(0x80, 0x43);
+ tsc1 = get_cycles();
+ lsb = inb(0x42);
+ msb = inb(0x42);
+ if (msb != val)
+ break;
+ }
+ *deltap = tsc1 - tsc0;
+ *tscp = tsc0;
+ *cnt = lsb | ((u16)msb << 8);
+
+ /*
+ * We require _some_ success, but the quality control
+ * will be based on the error terms on the TSC values.
+ */
+ return count > 5;
+}
+
+/*
* How many MSB values do we want to see? We aim for
* a maximum error rate of 500ppm (in practice the
* real error is much smaller), but refuse to spend
@@ -561,6 +595,94 @@ static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *de
#define MAX_QUICK_PIT_MS 50
#define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
+/*
+ * High latency version of quick_pit_calibrate() that works even if there is a
+ * high latency reading the PIT lsb/msb. This is needed if it takes longer than
+ * 12us to read the lsb/msb because then the error margin will never fall below
+ * 500ppm within 50ms.
+ */
+static unsigned long quick_pit_calibrate_hl(void)
+{
+ int i;
+ u64 tsc, delta;
+ unsigned long d1, d2;
+ u16 cnt0, cnt1, dc;
+
+ /* Re-start at 0xffff */
+ outb(0xff, 0x42);
+ outb(0xff, 0x42);
+
+ /*
+ * The PIT starts counting at the next edge, so we
+ * need to delay for a microsecond. The easiest way
+ * to do that is to just read back the 16-bit counter
+ * once from the PIT.
+ */
+ pit_verify_msb(0);
+
+ /*
+ * Iterate until the error is less than 500 ppm. The error margin due to
+ * the time to latch the counter is d1 + d2. The counter latches the
+ * current count which introduces a one tick error at the start and end.
+ * So the total error is d1 + d2 + 2 timer ticks. A timer tick is
+ * approximately the TSC delta divided by the timer delta. So the error
+ * margin is too high while:
+ * d1 + d2 + 2 * (delta / dc) >= delta >> 11
+ * => d1 + d2 >= (delta >> 11) - 2 * (delta / dc)
+ * => d1 + d2 >= (delta - 4096 * (delta / dc)) / 2048
+ * => (d1 + d2) * dc >= (dc * delta - 4096 * delta) / 2048
+ * => (d1 + d2) * dc >= (dc - 4096) * delta >> 11
+ */
+ if (pit_expect_msb_hl(0xff, &tsc, &d1, &cnt0)) {
+ for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) {
+ if (!pit_expect_msb_hl(0xff - i, &delta, &d2, &cnt1))
+ break;
+
+ dc = cnt0 - cnt1;
+ if (dc < 4096)
+ continue;
+
+ delta -= tsc;
+
+ if ((d1 + d2) * (u64)dc >= (dc - 4096) * delta >> 11)
+ continue;
+
+ /*
+ * Check the PIT one more time to verify that
+ * all TSC reads were stable wrt the PIT.
+ *
+ * This also guarantees serialization of the
+ * last cycle read ('d2') in pit_expect_msb.
+ */
+ if (!pit_verify_msb(0xfe - i))
+ break;
+ goto success;
+ }
+ }
+ pr_info("Fast TSC calibration (with latch) failed\n");
+ return 0;
+
+success:
+ /*
+ * Ok, if we get here, then we've seen the
+ * MSB of the PIT decrement 'i' times, and the
+ * error has shrunk to less than 500 ppm.
+ *
+ * As a result, we can depend on there not being
+ * any odd delays anywhere, and the TSC reads are
+ * reliable (within the error).
+ *
+ * kHz = ticks / time-in-seconds / 1000;
+ * kHz = ticks / (PIT count / PIT_TICK_RATE) / 1000
+ * kHz = delta / (dc / PIT_TICK_RATE) / 1000
+ * kHz = (delta * PIT_TICK_RATE) / (dc * 1000)
+ */
+ delta *= PIT_TICK_RATE;
+ do_div(delta, dc * 1000);
+ pr_info("Fast TSC calibration (with latch) using PIT\n");
+ return delta;
+}
+
static unsigned long quick_pit_calibrate(void)
{
int i;
@@ -598,10 +720,19 @@ static unsigned long quick_pit_calibrate(void)
if (!pit_expect_msb(0xff-i, &delta, &d2))
break;
+ delta -= tsc;
+
+ /*
+ * Extrapolate the error and switch to high-latency
+ * algorithm if the error will never be below 500 ppm.
+ */
+ if (i == 1 &&
+ d1 + d2 >= (delta * MAX_QUICK_PIT_ITERATIONS) >> 11)
+ return quick_pit_calibrate_hl();
+
/*
* Iterate until the error is less than 500 ppm
*/
- delta -= tsc;
if (d1+d2 >= delta >> 11)
continue;
--
1.9.1
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists