tsc.c 38.4 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

Alok Kataria's avatar
Alok Kataria committed
4
#include <linux/kernel.h>
Alok Kataria's avatar
Alok Kataria committed
5
#include <linux/sched.h>
6
#include <linux/sched/clock.h>
Alok Kataria's avatar
Alok Kataria committed
7
#include <linux/init.h>
8
#include <linux/export.h>
Alok Kataria's avatar
Alok Kataria committed
9
#include <linux/timer.h>
Alok Kataria's avatar
Alok Kataria committed
10
#include <linux/acpi_pmtmr.h>
11
#include <linux/cpufreq.h>
12 13 14
#include <linux/delay.h>
#include <linux/clocksource.h>
#include <linux/percpu.h>
15
#include <linux/timex.h>
16
#include <linux/static_key.h>
Alok Kataria's avatar
Alok Kataria committed
17 18

#include <asm/hpet.h>
19 20 21 22
#include <asm/timer.h>
#include <asm/vgtod.h>
#include <asm/time.h>
#include <asm/delay.h>
23
#include <asm/hypervisor.h>
24
#include <asm/nmi.h>
25
#include <asm/x86_init.h>
26
#include <asm/geode.h>
27
#include <asm/apic.h>
28
#include <asm/intel-family.h>
29
#include <asm/i8259.h>
30
#include <asm/uv/uv.h>
Alok Kataria's avatar
Alok Kataria committed
31

32
unsigned int __read_mostly cpu_khz;	/* TSC clocks / usec, not used here */
Alok Kataria's avatar
Alok Kataria committed
33
EXPORT_SYMBOL(cpu_khz);
34 35

unsigned int __read_mostly tsc_khz;
Alok Kataria's avatar
Alok Kataria committed
36 37
EXPORT_SYMBOL(tsc_khz);

38 39
#define KHZ	1000

Alok Kataria's avatar
Alok Kataria committed
40 41 42
/*
 * TSC can be unstable due to cpufreq or due to unsynced TSCs
 */
43
static int __read_mostly tsc_unstable;
Alok Kataria's avatar
Alok Kataria committed
44

45
static DEFINE_STATIC_KEY_FALSE(__use_tsc);
46

47
int tsc_clocksource_reliable;
48

49 50 51 52 53
static u32 art_to_tsc_numerator;
static u32 art_to_tsc_denominator;
static u64 art_to_tsc_offset;
struct clocksource *art_related_clocksource;

54
struct cyc2ns {
55 56
	struct cyc2ns_data data[2];	/*  0 + 2*16 = 32 */
	seqcount_t	   seq;		/* 32 + 4    = 36 */
57

58
}; /* fits one cacheline */
59

60
static DEFINE_PER_CPU_ALIGNED(struct cyc2ns, cyc2ns);
61

62
void __always_inline cyc2ns_read_begin(struct cyc2ns_data *data)
63
{
64
	int seq, idx;
65

66
	preempt_disable_notrace();
67

68 69 70
	do {
		seq = this_cpu_read(cyc2ns.seq.sequence);
		idx = seq & 1;
71

72 73 74
		data->cyc2ns_offset = this_cpu_read(cyc2ns.data[idx].cyc2ns_offset);
		data->cyc2ns_mul    = this_cpu_read(cyc2ns.data[idx].cyc2ns_mul);
		data->cyc2ns_shift  = this_cpu_read(cyc2ns.data[idx].cyc2ns_shift);
75

76
	} while (unlikely(seq != this_cpu_read(cyc2ns.seq.sequence)));
77 78
}

79
void __always_inline cyc2ns_read_end(void)
80
{
81
	preempt_enable_notrace();
82 83 84 85
}

/*
 * Accelerators for sched_clock()
86 87 88 89 90 91 92 93 94 95 96 97
 * convert from cycles(64bits) => nanoseconds (64bits)
 *  basic equation:
 *              ns = cycles / (freq / ns_per_sec)
 *              ns = cycles * (ns_per_sec / freq)
 *              ns = cycles * (10^9 / (cpu_khz * 10^3))
 *              ns = cycles * (10^6 / cpu_khz)
 *
 *      Then we use scaling math (suggested by george@mvista.com) to get:
 *              ns = cycles * (10^6 * SC / cpu_khz) / SC
 *              ns = cycles * cyc2ns_scale / SC
 *
 *      And since SC is a constant power of two, we can convert the div
98 99 100
 *  into a shift. The larger SC is, the more accurate the conversion, but
 *  cyc2ns_scale needs to be a 32-bit value so that 32-bit multiplication
 *  (64-bit result) can be used.
101
 *
102
 *  We can use khz divisor instead of mhz to keep a better precision.
103 104 105 106 107
 *  (mathieu.desnoyers@polymtl.ca)
 *
 *                      -johnstul@us.ibm.com "math is hard, lets go shopping!"
 */

108
static __always_inline unsigned long long cycles_2_ns(unsigned long long cyc)
109
{
110
	struct cyc2ns_data data;
111 112
	unsigned long long ns;

113
	cyc2ns_read_begin(&data);
114

115 116
	ns = data.cyc2ns_offset;
	ns += mul_u64_u32_shr(cyc, data.cyc2ns_mul, data.cyc2ns_shift);
117

118
	cyc2ns_read_end();
119

120 121 122
	return ns;
}

123
static void __set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now)
124
{
125
	unsigned long long ns_now;
126 127
	struct cyc2ns_data data;
	struct cyc2ns *c2n;
128

129 130
	ns_now = cycles_2_ns(tsc_now);

131 132 133 134 135
	/*
	 * Compute a new multiplier as per the above comment and ensure our
	 * time function is continuous; see the comment near struct
	 * cyc2ns_data.
	 */
136
	clocks_calc_mult_shift(&data.cyc2ns_mul, &data.cyc2ns_shift, khz,
137 138
			       NSEC_PER_MSEC, 0);

139 140 141 142 143 144
	/*
	 * cyc2ns_shift is exported via arch_perf_update_userpage() where it is
	 * not expected to be greater than 31 due to the original published
	 * conversion algorithm shifting a 32-bit value (now specifies a 64-bit
	 * value) - refer perf_event_mmap_page documentation in perf_event.h.
	 */
145 146 147
	if (data.cyc2ns_shift == 32) {
		data.cyc2ns_shift = 31;
		data.cyc2ns_mul >>= 1;
148 149
	}

150 151 152 153
	data.cyc2ns_offset = ns_now -
		mul_u64_u32_shr(tsc_now, data.cyc2ns_mul, data.cyc2ns_shift);

	c2n = per_cpu_ptr(&cyc2ns, cpu);
154

155 156 157 158
	raw_write_seqcount_latch(&c2n->seq);
	c2n->data[0] = data;
	raw_write_seqcount_latch(&c2n->seq);
	c2n->data[1] = data;
159 160 161 162 163 164 165 166 167 168 169
}

static void set_cyc2ns_scale(unsigned long khz, int cpu, unsigned long long tsc_now)
{
	unsigned long flags;

	local_irq_save(flags);
	sched_clock_idle_sleep_event();

	if (khz)
		__set_cyc2ns_scale(khz, cpu, tsc_now);
170

171
	sched_clock_idle_wakeup_event();
172 173
	local_irq_restore(flags);
}
174

175 176 177 178 179 180 181 182 183 184 185 186
/*
 * Initialize cyc2ns for boot cpu
 */
static void __init cyc2ns_init_boot_cpu(void)
{
	struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns);

	seqcount_init(&c2n->seq);
	__set_cyc2ns_scale(tsc_khz, smp_processor_id(), rdtsc());
}

/*
Dou Liyang's avatar
Dou Liyang committed
187
 * Secondary CPUs do not run through tsc_init(), so set up
188
 * all the scale factors for all CPUs, assuming the same
189
 * speed as the bootup CPU.
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206
 */
static void __init cyc2ns_init_secondary_cpus(void)
{
	unsigned int cpu, this_cpu = smp_processor_id();
	struct cyc2ns *c2n = this_cpu_ptr(&cyc2ns);
	struct cyc2ns_data *data = c2n->data;

	for_each_possible_cpu(cpu) {
		if (cpu != this_cpu) {
			seqcount_init(&c2n->seq);
			c2n = per_cpu_ptr(&cyc2ns, cpu);
			c2n->data[0] = data[0];
			c2n->data[1] = data[1];
		}
	}
}

Alok Kataria's avatar
Alok Kataria committed
207 208 209 210 211
/*
 * Scheduler clock - returns current time in nanosec units.
 */
u64 native_sched_clock(void)
{
212 213 214 215 216 217
	if (static_branch_likely(&__use_tsc)) {
		u64 tsc_now = rdtsc();

		/* return the value in ns */
		return cycles_2_ns(tsc_now);
	}
Alok Kataria's avatar
Alok Kataria committed
218 219 220 221 222 223 224

	/*
	 * Fall back to jiffies if there's no TSC available:
	 * ( But note that we still use it if the TSC is marked
	 *   unstable. We do this because unlike Time Of Day,
	 *   the scheduler clock tolerates small errors and it's
	 *   very important for it to be as fast as the platform
225
	 *   can achieve it. )
Alok Kataria's avatar
Alok Kataria committed
226 227
	 */

228 229
	/* No locking but a rare wrong value is not a big deal: */
	return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ);
Alok Kataria's avatar
Alok Kataria committed
230 231
}

232 233 234 235 236 237 238 239
/*
 * Generate a sched_clock if you already have a TSC value.
 */
u64 native_sched_clock_from_tsc(u64 tsc)
{
	return cycles_2_ns(tsc);
}

Alok Kataria's avatar
Alok Kataria committed
240 241 242 243 244 245 246
/* We need to define a real function for sched_clock, to override the
   weak default version */
#ifdef CONFIG_PARAVIRT
unsigned long long sched_clock(void)
{
	return paravirt_sched_clock();
}
247

248
bool using_native_sched_clock(void)
249
{
250
	return pv_ops.time.sched_clock == native_sched_clock;
251
}
Alok Kataria's avatar
Alok Kataria committed
252 253 254
#else
unsigned long long
sched_clock(void) __attribute__((alias("native_sched_clock")));
255

256
bool using_native_sched_clock(void) { return true; }
Alok Kataria's avatar
Alok Kataria committed
257 258 259 260 261 262 263 264 265 266 267
#endif

int check_tsc_unstable(void)
{
	return tsc_unstable;
}
EXPORT_SYMBOL_GPL(check_tsc_unstable);

#ifdef CONFIG_X86_TSC
int __init notsc_setup(char *str)
{
268
	mark_tsc_unstable("boot parameter notsc");
Alok Kataria's avatar
Alok Kataria committed
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
	return 1;
}
#else
/*
 * disable flag for tsc. Takes effect by clearing the TSC cpu flag
 * in cpu/common.c
 */
int __init notsc_setup(char *str)
{
	setup_clear_cpu_cap(X86_FEATURE_TSC);
	return 1;
}
#endif

__setup("notsc", notsc_setup);
Alok Kataria's avatar
Alok Kataria committed
284

285
static int no_sched_irq_time;
286
static int no_tsc_watchdog;
287

288 289 290 291
static int __init tsc_setup(char *str)
{
	if (!strcmp(str, "reliable"))
		tsc_clocksource_reliable = 1;
292 293
	if (!strncmp(str, "noirqtime", 9))
		no_sched_irq_time = 1;
294 295
	if (!strcmp(str, "unstable"))
		mark_tsc_unstable("boot parameter");
296 297
	if (!strcmp(str, "nowatchdog"))
		no_tsc_watchdog = 1;
298 299 300 301 302
	return 1;
}

__setup("tsc=", tsc_setup);

303 304
#define MAX_RETRIES		5
#define TSC_DEFAULT_THRESHOLD	0x20000
Alok Kataria's avatar
Alok Kataria committed
305 306

/*
307
 * Read TSC and the reference counters. Take care of any disturbances
Alok Kataria's avatar
Alok Kataria committed
308
 */
309
static u64 tsc_read_refs(u64 *p, int hpet)
Alok Kataria's avatar
Alok Kataria committed
310 311
{
	u64 t1, t2;
312
	u64 thresh = tsc_khz ? tsc_khz >> 5 : TSC_DEFAULT_THRESHOLD;
Alok Kataria's avatar
Alok Kataria committed
313 314 315 316 317
	int i;

	for (i = 0; i < MAX_RETRIES; i++) {
		t1 = get_cycles();
		if (hpet)
318
			*p = hpet_readl(HPET_COUNTER) & 0xFFFFFFFF;
Alok Kataria's avatar
Alok Kataria committed
319
		else
320
			*p = acpi_pm_read_early();
Alok Kataria's avatar
Alok Kataria committed
321
		t2 = get_cycles();
322
		if ((t2 - t1) < thresh)
Alok Kataria's avatar
Alok Kataria committed
323 324 325 326 327
			return t2;
	}
	return ULLONG_MAX;
}

328 329
/*
 * Calculate the TSC frequency from HPET reference
Alok Kataria's avatar
Alok Kataria committed
330
 */
331
static unsigned long calc_hpet_ref(u64 deltatsc, u64 hpet1, u64 hpet2)
Alok Kataria's avatar
Alok Kataria committed
332
{
333
	u64 tmp;
Alok Kataria's avatar
Alok Kataria committed
334

335 336 337 338 339
	if (hpet2 < hpet1)
		hpet2 += 0x100000000ULL;
	hpet2 -= hpet1;
	tmp = ((u64)hpet2 * hpet_readl(HPET_PERIOD));
	do_div(tmp, 1000000);
340
	deltatsc = div64_u64(deltatsc, tmp);
341 342 343 344 345 346 347 348 349 350

	return (unsigned long) deltatsc;
}

/*
 * Calculate the TSC frequency from PMTimer reference
 */
static unsigned long calc_pmtimer_ref(u64 deltatsc, u64 pm1, u64 pm2)
{
	u64 tmp;
Alok Kataria's avatar
Alok Kataria committed
351

352 353 354 355 356 357 358 359 360 361 362 363 364
	if (!pm1 && !pm2)
		return ULONG_MAX;

	if (pm2 < pm1)
		pm2 += (u64)ACPI_PM_OVRRUN;
	pm2 -= pm1;
	tmp = pm2 * 1000000000LL;
	do_div(tmp, PMTMR_TICKS_PER_SEC);
	do_div(deltatsc, tmp);

	return (unsigned long) deltatsc;
}

365
#define CAL_MS		10
366
#define CAL_LATCH	(PIT_TICK_RATE / (1000 / CAL_MS))
367 368 369
#define CAL_PIT_LOOPS	1000

#define CAL2_MS		50
370
#define CAL2_LATCH	(PIT_TICK_RATE / (1000 / CAL2_MS))
371 372
#define CAL2_PIT_LOOPS	5000

373

374 375 376 377 378 379 380
/*
 * Try to calibrate the TSC against the Programmable
 * Interrupt Timer and return the frequency of the TSC
 * in kHz.
 *
 * Return ULONG_MAX on failure to calibrate.
 */
381
static unsigned long pit_calibrate_tsc(u32 latch, unsigned long ms, int loopmin)
382 383 384 385 386
{
	u64 tsc, t1, t2, delta;
	unsigned long tscmin, tscmax;
	int pitcnt;

387 388 389 390 391 392 393 394 395 396 397 398 399 400
	if (!has_legacy_pic()) {
		/*
		 * Relies on tsc_early_delay_calibrate() to have given us semi
		 * usable udelay(), wait for the same 50ms we would have with
		 * the PIT loop below.
		 */
		udelay(10 * USEC_PER_MSEC);
		udelay(10 * USEC_PER_MSEC);
		udelay(10 * USEC_PER_MSEC);
		udelay(10 * USEC_PER_MSEC);
		udelay(10 * USEC_PER_MSEC);
		return ULONG_MAX;
	}

401 402 403 404 405 406 407 408 409
	/* Set the Gate high, disable speaker */
	outb((inb(0x61) & ~0x02) | 0x01, 0x61);

	/*
	 * Setup CTC channel 2* for mode 0, (interrupt on terminal
	 * count mode), binary count. Set the latch register to 50ms
	 * (LSB then MSB) to begin countdown.
	 */
	outb(0xb0, 0x43);
410 411
	outb(latch & 0xff, 0x42);
	outb(latch >> 8, 0x42);
412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431

	tsc = t1 = t2 = get_cycles();

	pitcnt = 0;
	tscmax = 0;
	tscmin = ULONG_MAX;
	while ((inb(0x61) & 0x20) == 0) {
		t2 = get_cycles();
		delta = t2 - tsc;
		tsc = t2;
		if ((unsigned long) delta < tscmin)
			tscmin = (unsigned int) delta;
		if ((unsigned long) delta > tscmax)
			tscmax = (unsigned int) delta;
		pitcnt++;
	}

	/*
	 * Sanity checks:
	 *
432
	 * If we were not able to read the PIT more than loopmin
433 434 435 436 437
	 * times, then we have been hit by a massive SMI
	 *
	 * If the maximum is 10 times larger than the minimum,
	 * then we got hit by an SMI as well.
	 */
438
	if (pitcnt < loopmin || tscmax > 10 * tscmin)
439 440 441 442
		return ULONG_MAX;

	/* Calculate the PIT value */
	delta = t2 - t1;
443
	do_div(delta, ms);
444 445 446
	return delta;
}

Linus Torvalds's avatar
Linus Torvalds committed
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481
/*
 * This reads the current MSB of the PIT counter, and
 * checks if we are running on sufficiently fast and
 * non-virtualized hardware.
 *
 * Our expectations are:
 *
 *  - the PIT is running at roughly 1.19MHz
 *
 *  - each IO is going to take about 1us on real hardware,
 *    but we allow it to be much faster (by a factor of 10) or
 *    _slightly_ slower (ie we allow up to a 2us read+counter
 *    update - anything else implies a unacceptably slow CPU
 *    or PIT for the fast calibration to work.
 *
 *  - with 256 PIT ticks to read the value, we have 214us to
 *    see the same MSB (and overhead like doing a single TSC
 *    read per MSB value etc).
 *
 *  - We're doing 2 reads per loop (LSB, MSB), and we expect
 *    them each to take about a microsecond on real hardware.
 *    So we expect a count value of around 100. But we'll be
 *    generous, and accept anything over 50.
 *
 *  - if the PIT is stuck, and we see *many* more reads, we
 *    return early (and the next caller of pit_expect_msb()
 *    then consider it a failure when they don't see the
 *    next expected value).
 *
 * These expectations mean that we know that we have seen the
 * transition from one expected value to another with a fairly
 * high accuracy, and we didn't miss any events. We can thus
 * use the TSC value at the transitions to calculate a pretty
 * good value for the TSC frequencty.
 */
482 483 484 485 486 487 488
static inline int pit_verify_msb(unsigned char val)
{
	/* Ignore LSB */
	inb(0x42);
	return inb(0x42) == val;
}

489
static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *deltap)
Linus Torvalds's avatar
Linus Torvalds committed
490
{
491
	int count;
492
	u64 tsc = 0, prev_tsc = 0;
Alok Kataria's avatar
Alok Kataria committed
493

Linus Torvalds's avatar
Linus Torvalds committed
494
	for (count = 0; count < 50000; count++) {
495
		if (!pit_verify_msb(val))
Linus Torvalds's avatar
Linus Torvalds committed
496
			break;
497
		prev_tsc = tsc;
498
		tsc = get_cycles();
Linus Torvalds's avatar
Linus Torvalds committed
499
	}
500
	*deltap = get_cycles() - prev_tsc;
501 502 503 504 505 506 507
	*tscp = tsc;

	/*
	 * We require _some_ success, but the quality control
	 * will be based on the error terms on the TSC values.
	 */
	return count > 5;
Linus Torvalds's avatar
Linus Torvalds committed
508 509 510
}

/*
511 512 513
 * How many MSB values do we want to see? We aim for
 * a maximum error rate of 500ppm (in practice the
 * real error is much smaller), but refuse to spend
514
 * more than 50ms on it.
Linus Torvalds's avatar
Linus Torvalds committed
515
 */
516
#define MAX_QUICK_PIT_MS 50
517
#define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
Alok Kataria's avatar
Alok Kataria committed
518

Linus Torvalds's avatar
Linus Torvalds committed
519 520
static unsigned long quick_pit_calibrate(void)
{
521 522 523 524
	int i;
	u64 tsc, delta;
	unsigned long d1, d2;

525 526 527
	if (!has_legacy_pic())
		return 0;

Linus Torvalds's avatar
Linus Torvalds committed
528
	/* Set the Gate high, disable speaker */
Alok Kataria's avatar
Alok Kataria committed
529 530
	outb((inb(0x61) & ~0x02) | 0x01, 0x61);

Linus Torvalds's avatar
Linus Torvalds committed
531 532 533 534 535 536 537 538 539
	/*
	 * Counter 2, mode 0 (one-shot), binary count
	 *
	 * NOTE! Mode 2 decrements by two (and then the
	 * output is flipped each time, giving the same
	 * final output frequency as a decrement-by-one),
	 * so mode 0 is much better when looking at the
	 * individual counts.
	 */
Alok Kataria's avatar
Alok Kataria committed
540 541
	outb(0xb0, 0x43);

Linus Torvalds's avatar
Linus Torvalds committed
542 543 544 545
	/* Start at 0xffff */
	outb(0xff, 0x42);
	outb(0xff, 0x42);

546 547 548 549 550 551
	/*
	 * The PIT starts counting at the next edge, so we
	 * need to delay for a microsecond. The easiest way
	 * to do that is to just read back the 16-bit counter
	 * once from the PIT.
	 */
552
	pit_verify_msb(0);
553

554 555 556 557 558
	if (pit_expect_msb(0xff, &tsc, &d1)) {
		for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) {
			if (!pit_expect_msb(0xff-i, &delta, &d2))
				break;

559 560 561 562 563 564 565 566 567 568
			delta -= tsc;

			/*
			 * Extrapolate the error and fail fast if the error will
			 * never be below 500 ppm.
			 */
			if (i == 1 &&
			    d1 + d2 >= (delta * MAX_QUICK_PIT_ITERATIONS) >> 11)
				return 0;

569 570 571
			/*
			 * Iterate until the error is less than 500 ppm
			 */
572 573 574 575 576 577 578 579 580 581 582 583 584
			if (d1+d2 >= delta >> 11)
				continue;

			/*
			 * Check the PIT one more time to verify that
			 * all TSC reads were stable wrt the PIT.
			 *
			 * This also guarantees serialization of the
			 * last cycle read ('d2') in pit_expect_msb.
			 */
			if (!pit_verify_msb(0xfe - i))
				break;
			goto success;
Linus Torvalds's avatar
Linus Torvalds committed
585 586
		}
	}
587
	pr_info("Fast TSC calibration failed\n");
Linus Torvalds's avatar
Linus Torvalds committed
588
	return 0;
589 590 591 592 593 594 595 596 597

success:
	/*
	 * Ok, if we get here, then we've seen the
	 * MSB of the PIT decrement 'i' times, and the
	 * error has shrunk to less than 500 ppm.
	 *
	 * As a result, we can depend on there not being
	 * any odd delays anywhere, and the TSC reads are
598
	 * reliable (within the error).
599 600 601 602 603 604 605
	 *
	 * kHz = ticks / time-in-seconds / 1000;
	 * kHz = (t2 - t1) / (I * 256 / PIT_TICK_RATE) / 1000
	 * kHz = ((t2 - t1) * PIT_TICK_RATE) / (I * 256 * 1000)
	 */
	delta *= PIT_TICK_RATE;
	do_div(delta, i*256*1000);
606
	pr_info("Fast TSC calibration using PIT\n");
607
	return delta;
Linus Torvalds's avatar
Linus Torvalds committed
608
}
609

Alok Kataria's avatar
Alok Kataria committed
610
/**
611 612
 * native_calibrate_tsc
 * Determine TSC frequency via CPUID, else return 0.
Alok Kataria's avatar
Alok Kataria committed
613
 */
614
unsigned long native_calibrate_tsc(void)
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636
{
	unsigned int eax_denominator, ebx_numerator, ecx_hz, edx;
	unsigned int crystal_khz;

	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 0;

	if (boot_cpu_data.cpuid_level < 0x15)
		return 0;

	eax_denominator = ebx_numerator = ecx_hz = edx = 0;

	/* CPUID 15H TSC/Crystal ratio, plus optionally Crystal Hz */
	cpuid(0x15, &eax_denominator, &ebx_numerator, &ecx_hz, &edx);

	if (ebx_numerator == 0 || eax_denominator == 0)
		return 0;

	crystal_khz = ecx_hz / 1000;

	if (crystal_khz == 0) {
		switch (boot_cpu_data.x86_model) {
637 638
		case INTEL_FAM6_SKYLAKE_MOBILE:
		case INTEL_FAM6_SKYLAKE_DESKTOP:
639 640
		case INTEL_FAM6_KABYLAKE_MOBILE:
		case INTEL_FAM6_KABYLAKE_DESKTOP:
641 642
			crystal_khz = 24000;	/* 24.0 MHz */
			break;
643
		case INTEL_FAM6_ATOM_GOLDMONT_X:
644 645
			crystal_khz = 25000;	/* 25.0 MHz */
			break;
646
		case INTEL_FAM6_ATOM_GOLDMONT:
647 648
			crystal_khz = 19200;	/* 19.2 MHz */
			break;
649 650 651
		}
	}

652 653
	if (crystal_khz == 0)
		return 0;
654 655 656 657 658 659 660
	/*
	 * TSC frequency determined by CPUID is a "hardware reported"
	 * frequency and is the most accurate one so far we have. This
	 * is considered a known frequency.
	 */
	setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);

661 662 663 664 665 666 667
	/*
	 * For Atom SoCs TSC is the only reliable clocksource.
	 * Mark TSC reliable so no watchdog on it.
	 */
	if (boot_cpu_data.x86_model == INTEL_FAM6_ATOM_GOLDMONT)
		setup_force_cpu_cap(X86_FEATURE_TSC_RELIABLE);

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687
	return crystal_khz * ebx_numerator / eax_denominator;
}

static unsigned long cpu_khz_from_cpuid(void)
{
	unsigned int eax_base_mhz, ebx_max_mhz, ecx_bus_mhz, edx;

	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 0;

	if (boot_cpu_data.cpuid_level < 0x16)
		return 0;

	eax_base_mhz = ebx_max_mhz = ecx_bus_mhz = edx = 0;

	cpuid(0x16, &eax_base_mhz, &ebx_max_mhz, &ecx_bus_mhz, &edx);

	return eax_base_mhz * 1000;
}

688 689 690
/*
 * calibrate cpu using pit, hpet, and ptimer methods. They are available
 * later in boot after acpi is initialized.
691
 */
692
static unsigned long pit_hpet_ptimer_calibrate_cpu(void)
Alok Kataria's avatar
Alok Kataria committed
693
{
694
	u64 tsc1, tsc2, delta, ref1, ref2;
695
	unsigned long tsc_pit_min = ULONG_MAX, tsc_ref_min = ULONG_MAX;
696
	unsigned long flags, latch, ms;
697
	int hpet = is_hpet_enabled(), i, loopmin;
Alok Kataria's avatar
Alok Kataria committed
698

699 700 701 702 703 704 705 706 707 708 709
	/*
	 * Run 5 calibration loops to get the lowest frequency value
	 * (the best estimate). We use two different calibration modes
	 * here:
	 *
	 * 1) PIT loop. We set the PIT Channel 2 to oneshot mode and
	 * load a timeout of 50ms. We read the time right after we
	 * started the timer and wait until the PIT count down reaches
	 * zero. In each wait loop iteration we read the TSC and check
	 * the delta to the previous read. We keep track of the min
	 * and max values of that delta. The delta is mostly defined
710 711
	 * by the IO time of the PIT access, so we can detect when
	 * any disturbance happened between the two reads. If the
712 713 714 715 716 717
	 * maximum time is significantly larger than the minimum time,
	 * then we discard the result and have another try.
	 *
	 * 2) Reference counter. If available we use the HPET or the
	 * PMTIMER as a reference to check the sanity of that value.
	 * We use separate TSC readouts and check inside of the
718
	 * reference read for any possible disturbance. We dicard
719 720 721 722
	 * disturbed values here as well. We do that around the PIT
	 * calibration delay loop as we have to wait for a certain
	 * amount of time anyway.
	 */
723 724 725 726 727 728 729

	/* Preset PIT loop values */
	latch = CAL_LATCH;
	ms = CAL_MS;
	loopmin = CAL_PIT_LOOPS;

	for (i = 0; i < 3; i++) {
730
		unsigned long tsc_pit_khz;
731 732 733

		/*
		 * Read the start value and the reference count of
734 735 736
		 * hpet/pmtimer when available. Then do the PIT
		 * calibration, which will take at least 50ms, and
		 * read the end value.
737
		 */
738
		local_irq_save(flags);
739
		tsc1 = tsc_read_refs(&ref1, hpet);
740
		tsc_pit_khz = pit_calibrate_tsc(latch, ms, loopmin);
741
		tsc2 = tsc_read_refs(&ref2, hpet);
742 743
		local_irq_restore(flags);

744 745
		/* Pick the lowest PIT TSC calibration so far */
		tsc_pit_min = min(tsc_pit_min, tsc_pit_khz);
746 747

		/* hpet or pmtimer available ? */
748
		if (ref1 == ref2)
749 750
			continue;

751
		/* Check, whether the sampling was disturbed */
752 753 754 755
		if (tsc1 == ULLONG_MAX || tsc2 == ULLONG_MAX)
			continue;

		tsc2 = (tsc2 - tsc1) * 1000000LL;
756
		if (hpet)
757
			tsc2 = calc_hpet_ref(tsc2, ref1, ref2);
758
		else
759
			tsc2 = calc_pmtimer_ref(tsc2, ref1, ref2);
760 761

		tsc_ref_min = min(tsc_ref_min, (unsigned long) tsc2);
762 763 764 765 766 767 768 769 770 771 772 773

		/* Check the reference deviation */
		delta = ((u64) tsc_pit_min) * 100;
		do_div(delta, tsc_ref_min);

		/*
		 * If both calibration results are inside a 10% window
		 * then we can be sure, that the calibration
		 * succeeded. We break out of the loop right away. We
		 * use the reference value, as it is more precise.
		 */
		if (delta >= 90 && delta <= 110) {
774 775
			pr_info("PIT calibration matches %s. %d loops\n",
				hpet ? "HPET" : "PMTIMER", i + 1);
776
			return tsc_ref_min;
777 778
		}

779 780 781 782 783 784 785 786 787 788 789
		/*
		 * Check whether PIT failed more than once. This
		 * happens in virtualized environments. We need to
		 * give the virtual PC a slightly longer timeframe for
		 * the HPET/PMTIMER to make the result precise.
		 */
		if (i == 1 && tsc_pit_min == ULONG_MAX) {
			latch = CAL2_LATCH;
			ms = CAL2_MS;
			loopmin = CAL2_PIT_LOOPS;
		}
790
	}
Alok Kataria's avatar
Alok Kataria committed
791 792

	/*
793
	 * Now check the results.
Alok Kataria's avatar
Alok Kataria committed
794
	 */
795 796
	if (tsc_pit_min == ULONG_MAX) {
		/* PIT gave no useful value */
797
		pr_warn("Unable to calibrate against PIT\n");
798 799

		/* We don't have an alternative source, disable TSC */
800
		if (!hpet && !ref1 && !ref2) {
801
			pr_notice("No reference (HPET/PMTIMER) available\n");
802 803 804 805 806
			return 0;
		}

		/* The alternative source failed as well, disable TSC */
		if (tsc_ref_min == ULONG_MAX) {
807
			pr_warn("HPET/PMTIMER calibration failed\n");
808 809 810 811
			return 0;
		}

		/* Use the alternative source */
812 813
		pr_info("using %s reference calibration\n",
			hpet ? "HPET" : "PMTIMER");
814 815 816

		return tsc_ref_min;
	}
Alok Kataria's avatar
Alok Kataria committed
817

818
	/* We don't have an alternative source, use the PIT calibration value */
819
	if (!hpet && !ref1 && !ref2) {
820
		pr_info("Using PIT calibration value\n");
821
		return tsc_pit_min;
Alok Kataria's avatar
Alok Kataria committed
822 823
	}

824 825
	/* The alternative source failed, use the PIT calibration value */
	if (tsc_ref_min == ULONG_MAX) {
826
		pr_warn("HPET/PMTIMER calibration failed. Using PIT calibration.\n");
827
		return tsc_pit_min;
Alok Kataria's avatar
Alok Kataria committed
828 829
	}

830 831 832
	/*
	 * The calibration values differ too much. In doubt, we use
	 * the PIT value as we know that there are PMTIMERs around
833
	 * running at double speed. At least we let the user know:
834
	 */
835 836 837
	pr_warn("PIT calibration deviates from %s: %lu %lu\n",
		hpet ? "HPET" : "PMTIMER", tsc_pit_min, tsc_ref_min);
	pr_info("Using PIT calibration value\n");
838
	return tsc_pit_min;
Alok Kataria's avatar
Alok Kataria committed
839 840
}

841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861
/**
 * native_calibrate_cpu_early - can calibrate the cpu early in boot
 */
unsigned long native_calibrate_cpu_early(void)
{
	unsigned long flags, fast_calibrate = cpu_khz_from_cpuid();

	if (!fast_calibrate)
		fast_calibrate = cpu_khz_from_msr();
	if (!fast_calibrate) {
		local_irq_save(flags);
		fast_calibrate = quick_pit_calibrate();
		local_irq_restore(flags);
	}
	return fast_calibrate;
}


/**
 * native_calibrate_cpu - calibrate the cpu
 */
862
static unsigned long native_calibrate_cpu(void)
863 864 865 866 867 868 869 870 871
{
	unsigned long tsc_freq = native_calibrate_cpu_early();

	if (!tsc_freq)
		tsc_freq = pit_hpet_ptimer_calibrate_cpu();

	return tsc_freq;
}

872
void recalibrate_cpu_khz(void)
Alok Kataria's avatar
Alok Kataria committed
873 874 875 876
{
#ifndef CONFIG_SMP
	unsigned long cpu_khz_old = cpu_khz;

877
	if (!boot_cpu_has(X86_FEATURE_TSC))
878
		return;
879

880
	cpu_khz = x86_platform.calibrate_cpu();
881
	tsc_khz = x86_platform.calibrate_tsc();
882 883
	if (tsc_khz == 0)
		tsc_khz = cpu_khz;
884 885
	else if (abs(cpu_khz - tsc_khz) * 10 > tsc_khz)
		cpu_khz = tsc_khz;
886 887
	cpu_data(0).loops_per_jiffy = cpufreq_scale(cpu_data(0).loops_per_jiffy,
						    cpu_khz_old, cpu_khz);
Alok Kataria's avatar
Alok Kataria committed
888 889 890 891 892
#endif
}

EXPORT_SYMBOL(recalibrate_cpu_khz);

893

894 895
static unsigned long long cyc2ns_suspend;

896
void tsc_save_sched_clock_state(void)
897
{
898
	if (!sched_clock_stable())
899 900 901 902 903 904 905 906 907 908 909 910 911
		return;

	cyc2ns_suspend = sched_clock();
}

/*
 * Even on processors with invariant TSC, TSC gets reset in some the
 * ACPI system sleep states. And in some systems BIOS seem to reinit TSC to
 * arbitrary value (still sync'd across cpu's) during resume from such sleep
 * states. To cope up with this, recompute the cyc2ns_offset for each cpu so
 * that sched_clock() continues from the point where it was left off during
 * suspend.
 */
912
void tsc_restore_sched_clock_state(void)
913 914 915 916 917
{
	unsigned long long offset;
	unsigned long flags;
	int cpu;

918
	if (!sched_clock_stable())
919 920 921 922
		return;

	local_irq_save(flags);

923
	/*
924
	 * We're coming out of suspend, there's no concurrency yet; don't
925 926 927 928 929 930 931
	 * bother being nice about the RCU stuff, just write to both
	 * data fields.
	 */

	this_cpu_write(cyc2ns.data[0].cyc2ns_offset, 0);
	this_cpu_write(cyc2ns.data[1].cyc2ns_offset, 0);

932 933
	offset = cyc2ns_suspend - sched_clock();

934 935 936 937
	for_each_possible_cpu(cpu) {
		per_cpu(cyc2ns.data[0].cyc2ns_offset, cpu) = offset;
		per_cpu(cyc2ns.data[1].cyc2ns_offset, cpu) = offset;
	}
938 939 940 941

	local_irq_restore(flags);
}

942
#ifdef CONFIG_CPU_FREQ
943 944
/*
 * Frequency scaling support. Adjust the TSC based timer when the CPU frequency
945 946
 * changes.
 *
947 948
 * NOTE: On SMP the situation is not fixable in general, so simply mark the TSC
 * as unstable and give up in those cases.
949 950 951 952 953 954 955 956 957 958 959 960 961 962
 *
 * Should fix up last_tsc too. Currently gettimeofday in the
 * first tick after the change will be slightly wrong.
 */

static unsigned int  ref_freq;
static unsigned long loops_per_jiffy_ref;
static unsigned long tsc_khz_ref;

static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct cpufreq_freqs *freq = data;

963 964 965 966
	if (num_online_cpus() > 1) {
		mark_tsc_unstable("cpufreq changes on SMP");
		return 0;
	}
967 968 969

	if (!ref_freq) {
		ref_freq = freq->old;
970
		loops_per_jiffy_ref = boot_cpu_data.loops_per_jiffy;
971 972
		tsc_khz_ref = tsc_khz;
	}
973

974
	if ((val == CPUFREQ_PRECHANGE  && freq->old < freq->new) ||
975 976 977
	    (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {
		boot_cpu_data.loops_per_jiffy =
			cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new);
978 979 980 981 982

		tsc_khz = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new);
		if (!(freq->flags & CPUFREQ_CONST_LOOPS))
			mark_tsc_unstable("cpufreq changes");

983
		set_cyc2ns_scale(tsc_khz, freq->policy->cpu, rdtsc());
Peter Zijlstra's avatar
Peter Zijlstra committed
984
	}
985 986 987 988 989 990 991 992

	return 0;
}

static struct notifier_block time_cpufreq_notifier_block = {
	.notifier_call  = time_cpufreq_notifier
};

993
static int __init cpufreq_register_tsc_scaling(void)
994
{
995
	if (!boot_cpu_has(X86_FEATURE_TSC))
996 997 998
		return 0;
	if (boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
		return 0;
999 1000 1001 1002 1003
	cpufreq_register_notifier(&time_cpufreq_notifier_block,
				CPUFREQ_TRANSITION_NOTIFIER);
	return 0;
}

1004
core_initcall(cpufreq_register_tsc_scaling);
1005 1006

#endif /* CONFIG_CPU_FREQ */
1007

1008 1009 1010 1011 1012 1013 1014
#define ART_CPUID_LEAF (0x15)
#define ART_MIN_DENOMINATOR (1)


/*
 * If ART is present detect the numerator:denominator to convert to TSC
 */
Dou Liyang's avatar