/* * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * The contents of this file constitute Original Code as defined in and * are subject to the Apple Public Source License Version 1.1 (the * "License"). You may not use this file except in compliance with the * License. Please obtain a copy of the License at * http://www.apple.com/publicsource and read it before using this file. * * This Original Code and all software distributed under the License are * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the * License for the specific language governing rights and limitations * under the License. * * @APPLE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * File: i386/rtclock.c * Purpose: Routines for handling the machine dependent * real-time clock. Historically, this clock is * generated by the Intel 8254 Programmable Interval * Timer, but local apic timers are now used for * this purpose with the master time reference being * the cpu clock counted by the timestamp MSR. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for kernel_map */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MAX(a,b) (((a)>(b))?(a):(b)) #define MIN(a,b) (((a)>(b))?(b):(a)) #define NSEC_PER_HZ (NSEC_PER_SEC / 100) /* nsec per tick */ #define UI_CPUFREQ_ROUNDING_FACTOR 10000000 int sysclk_config(void); int sysclk_init(void); kern_return_t sysclk_gettime( mach_timespec_t *cur_time); kern_return_t sysclk_getattr( clock_flavor_t flavor, clock_attr_t attr, mach_msg_type_number_t *count); void sysclk_setalarm( mach_timespec_t *alarm_time); /* * Lists of clock routines. */ struct clock_ops sysclk_ops = { sysclk_config, sysclk_init, sysclk_gettime, 0, sysclk_getattr, 0, sysclk_setalarm, }; int calend_config(void); int calend_init(void); kern_return_t calend_gettime( mach_timespec_t *cur_time); kern_return_t calend_getattr( clock_flavor_t flavor, clock_attr_t attr, mach_msg_type_number_t *count); struct clock_ops calend_ops = { calend_config, calend_init, calend_gettime, 0, calend_getattr, 0, 0, }; /* local data declarations */ static clock_timer_func_t rtclock_timer_expire; static timer_call_data_t rtclock_alarm_timer; static void rtclock_alarm_expire( timer_call_param_t p0, timer_call_param_t p1); struct { mach_timespec_t calend_offset; boolean_t calend_is_set; int64_t calend_adjtotal; int32_t calend_adjdelta; uint32_t boottime; mach_timebase_info_data_t timebase_const; decl_simple_lock_data(,lock) /* real-time clock device lock */ } rtclock; boolean_t rtc_initialized = FALSE; clock_res_t rtc_intr_nsec = NSEC_PER_HZ; /* interrupt res */ uint64_t rtc_cycle_count; /* clocks in 1/20th second */ uint64_t rtc_cyc_per_sec; /* processor cycles per sec */ uint32_t rtc_boot_frequency; /* provided by 1st speed-step */ uint32_t rtc_quant_scale; /* clock to nanos multiplier */ uint32_t rtc_quant_shift; /* clock to nanos right shift */ uint64_t rtc_decrementer_min; static mach_timebase_info_data_t rtc_lapic_scale; /* nsec to lapic count */ /* * Macros to lock/unlock real-time clock data. */ #define RTC_INTRS_OFF(s) \ (s) = splclock() #define RTC_INTRS_ON(s) \ splx(s) #define RTC_LOCK(s) \ MACRO_BEGIN \ RTC_INTRS_OFF(s); \ simple_lock(&rtclock.lock); \ MACRO_END #define RTC_UNLOCK(s) \ MACRO_BEGIN \ simple_unlock(&rtclock.lock); \ RTC_INTRS_ON(s); \ MACRO_END /* * i8254 control. ** MONUMENT ** * * The i8254 is a traditional PC device with some arbitrary characteristics. * Basically, it is a register that counts at a fixed rate and can be * programmed to generate an interrupt every N counts. The count rate is * clknum counts per sec (see pit.h), historically 1193167=14.318MHz/12 * but the more accurate value is 1193182=14.31818MHz/12. [14.31818 MHz being * the master crystal oscillator reference frequency since the very first PC.] * Various constants are computed based on this value, and we calculate * them at init time for execution efficiency. To obtain sufficient * accuracy, some of the calculation are most easily done in floating * point and then converted to int. * */ /* * Forward decl. */ static uint64_t rtc_set_cyc_per_sec(uint64_t cycles); uint64_t rtc_nanotime_read(void); /* * create_mul_quant_GHZ * create a constant used to multiply the TSC by to convert to nanoseconds. * This is a 32 bit number and the TSC *MUST* have a frequency higher than * 1000Mhz for this routine to work. * * The theory here is that we know how many TSCs-per-sec the processor runs at. * Normally to convert this to nanoseconds you would multiply the current * timestamp by 1000000000 (a billion) then divide by TSCs-per-sec. * Unfortunatly the TSC is 64 bits which would leave us with 96 bit intermediate * results from the multiply that must be divided by. * Usually thats * uint96 = tsc * numer * nanos = uint96 / denom * Instead, we create this quant constant and it becomes the numerator, * the denominator can then be 0x100000000 which makes our division as simple as * forgetting the lower 32 bits of the result. We can also pass this number to * user space as the numer and pass 0xFFFFFFFF (RTC_FAST_DENOM) as the denom to * convert raw counts * to nanos. The difference is so small as to be * undetectable by anything. * * Unfortunatly we can not do this for sub GHZ processors. In this case, all * we do is pass the CPU speed in raw as the denom and we pass in 1000000000 * as the numerator. No short cuts allowed */ #define RTC_FAST_DENOM 0xFFFFFFFF inline static uint32_t create_mul_quant_GHZ(int shift, uint32_t quant) { return (uint32_t)((((uint64_t)NSEC_PER_SEC/20) << shift) / quant); } /* * This routine takes a value of raw TSC ticks and applies the passed mul_quant * generated by create_mul_quant() This is our internal routine for creating * nanoseconds. * Since we don't really have uint96_t this routine basically does this.... * uint96_t intermediate = (*value) * scale * return (intermediate >> 32) */ inline static uint64_t fast_get_nano_from_abs(uint64_t value, int scale) { asm (" movl %%edx,%%esi \n\t" " mull %%ecx \n\t" " movl %%edx,%%edi \n\t" " movl %%esi,%%eax \n\t" " mull %%ecx \n\t" " xorl %%ecx,%%ecx \n\t" " addl %%edi,%%eax \n\t" " adcl %%ecx,%%edx " : "+A" (value) : "c" (scale) : "%esi", "%edi"); return value; } /* * This routine basically does this... * ts.tv_sec = nanos / 1000000000; create seconds * ts.tv_nsec = nanos % 1000000000; create remainder nanos */ inline static mach_timespec_t nanos_to_timespec(uint64_t nanos) { union { mach_timespec_t ts; uint64_t u64; } ret; ret.u64 = nanos; asm volatile("divl %1" : "+A" (ret.u64) : "r" (NSEC_PER_SEC)); return ret.ts; } /* * The following two routines perform the 96 bit arithmetic we need to * convert generic absolute<->nanoseconds * The multiply routine takes a uint64_t and a uint32_t and returns the result * in a uint32_t[3] array. * The divide routine takes this uint32_t[3] array and divides it by a uint32_t * returning a uint64_t */ inline static void longmul(uint64_t *abstime, uint32_t multiplicand, uint32_t *result) { asm volatile( " pushl %%ebx \n\t" " movl %%eax,%%ebx \n\t" " movl (%%eax),%%eax \n\t" " mull %%ecx \n\t" " xchg %%eax,%%ebx \n\t" " pushl %%edx \n\t" " movl 4(%%eax),%%eax \n\t" " mull %%ecx \n\t" " movl %2,%%ecx \n\t" " movl %%ebx,(%%ecx) \n\t" " popl %%ebx \n\t" " addl %%ebx,%%eax \n\t" " popl %%ebx \n\t" " movl %%eax,4(%%ecx) \n\t" " adcl $0,%%edx \n\t" " movl %%edx,8(%%ecx) // and save it" : : "a"(abstime), "c"(multiplicand), "m"(result)); } inline static uint64_t longdiv(uint32_t *numer, uint32_t denom) { uint64_t result; asm volatile( " pushl %%ebx \n\t" " movl %%eax,%%ebx \n\t" " movl 8(%%eax),%%edx \n\t" " movl 4(%%eax),%%eax \n\t" " divl %%ecx \n\t" " xchg %%ebx,%%eax \n\t" " movl (%%eax),%%eax \n\t" " divl %%ecx \n\t" " xchg %%ebx,%%edx \n\t" " popl %%ebx \n\t" : "=A"(result) : "a"(numer),"c"(denom)); return result; } /* * Enable or disable timer 2. * Port 0x61 controls timer 2: * bit 0 gates the clock, * bit 1 gates output to speaker. */ inline static void enable_PIT2(void) { asm volatile( " inb $0x61,%%al \n\t" " and $0xFC,%%al \n\t" " or $1,%%al \n\t" " outb %%al,$0x61 \n\t" : : : "%al" ); } inline static void disable_PIT2(void) { asm volatile( " inb $0x61,%%al \n\t" " and $0xFC,%%al \n\t" " outb %%al,$0x61 \n\t" : : : "%al" ); } inline static void set_PIT2(int value) { /* * First, tell the clock we are going to write 16 bits to the counter * and enable one-shot mode (command 0xB8 to port 0x43) * Then write the two bytes into the PIT2 clock register (port 0x42). * Loop until the value is "realized" in the clock, * this happens on the next tick. */ asm volatile( " movb $0xB8,%%al \n\t" " outb %%al,$0x43 \n\t" " movb %%dl,%%al \n\t" " outb %%al,$0x42 \n\t" " movb %%dh,%%al \n\t" " outb %%al,$0x42 \n" "1: inb $0x42,%%al \n\t" " inb $0x42,%%al \n\t" " cmp %%al,%%dh \n\t" " jne 1b" : : "d"(value) : "%al"); } inline static uint64_t get_PIT2(unsigned int *value) { register uint64_t result; /* * This routine first latches the time (command 0x80 to port 0x43), * then gets the time stamp so we know how long the read will take later. * Read (from port 0x42) and return the current value of the timer. */ asm volatile( " xorl %%ecx,%%ecx \n\t" " movb $0x80,%%al \n\t" " outb %%al,$0x43 \n\t" " rdtsc \n\t" " pushl %%eax \n\t" " inb $0x42,%%al \n\t" " movb %%al,%%cl \n\t" " inb $0x42,%%al \n\t" " movb %%al,%%ch \n\t" " popl %%eax " : "=A"(result), "=c"(*value)); return result; } /* * timeRDTSC() * This routine sets up PIT counter 2 to count down 1/20 of a second. * It pauses until the value is latched in the counter * and then reads the time stamp counter to return to the caller. */ static uint64_t timeRDTSC(void) { int attempts = 0; uint64_t latchTime; uint64_t saveTime,intermediate; unsigned int timerValue, lastValue; boolean_t int_enabled; /* * Table of correction factors to account for * - timer counter quantization errors, and * - undercounts 0..5 */ #define SAMPLE_CLKS_EXACT (((double) CLKNUM) / 20.0) #define SAMPLE_CLKS_INT ((int) CLKNUM / 20) #define SAMPLE_NSECS (2000000000LL) #define SAMPLE_MULTIPLIER (((double)SAMPLE_NSECS)*SAMPLE_CLKS_EXACT) #define ROUND64(x) ((uint64_t)((x) + 0.5)) uint64_t scale[6] = { ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-0)), ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-1)), ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-2)), ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-3)), ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-4)), ROUND64(SAMPLE_MULTIPLIER/(double)(SAMPLE_CLKS_INT-5)) }; int_enabled = ml_set_interrupts_enabled(FALSE); restart: if (attempts >= 2) panic("timeRDTSC() calibation failed with %d attempts\n", attempts); attempts++; enable_PIT2(); // turn on PIT2 set_PIT2(0); // reset timer 2 to be zero latchTime = rdtsc64(); // get the time stamp to time latchTime = get_PIT2(&timerValue) - latchTime; // time how long this takes set_PIT2(SAMPLE_CLKS_INT); // set up the timer for (almost) 1/20th a second saveTime = rdtsc64(); // now time how long a 20th a second is... get_PIT2(&lastValue); get_PIT2(&lastValue); // read twice, first value may be unreliable do { intermediate = get_PIT2(&timerValue); if (timerValue > lastValue) { printf("Hey we are going backwards! %u -> %u, restarting timing\n", timerValue,lastValue); set_PIT2(0); disable_PIT2(); goto restart; } lastValue = timerValue; } while (timerValue > 5); kprintf("timerValue %d\n",timerValue); kprintf("intermediate 0x%016llx\n",intermediate); kprintf("saveTime 0x%016llx\n",saveTime); intermediate -= saveTime; // raw count for about 1/20 second intermediate *= scale[timerValue]; // rescale measured time spent intermediate /= SAMPLE_NSECS; // so its exactly 1/20 a second intermediate += latchTime; // add on our save fudge set_PIT2(0); // reset timer 2 to be zero disable_PIT2(); // turn off PIT 2 ml_set_interrupts_enabled(int_enabled); return intermediate; } static uint64_t tsc_to_nanoseconds(uint64_t abstime) { uint32_t numer; uint32_t denom; uint32_t intermediate[3]; numer = rtclock.timebase_const.numer; denom = rtclock.timebase_const.denom; if (denom == RTC_FAST_DENOM) { abstime = fast_get_nano_from_abs(abstime, numer); } else { longmul(&abstime, numer, intermediate); abstime = longdiv(intermediate, denom); } return abstime; } inline static mach_timespec_t tsc_to_timespec(void) { uint64_t currNanos; currNanos = rtc_nanotime_read(); return nanos_to_timespec(currNanos); } #define DECREMENTER_MAX UINT_MAX static uint32_t deadline_to_decrementer( uint64_t deadline, uint64_t now) { uint64_t delta; if (deadline <= now) return rtc_decrementer_min; else { delta = deadline - now; return MIN(MAX(rtc_decrementer_min,delta),DECREMENTER_MAX); } } static inline uint64_t lapic_time_countdown(uint32_t initial_count) { boolean_t state; uint64_t start_time; uint64_t stop_time; lapic_timer_count_t count; state = ml_set_interrupts_enabled(FALSE); lapic_set_timer(FALSE, one_shot, divide_by_1, initial_count); start_time = rdtsc64(); do { lapic_get_timer(NULL, NULL, NULL, &count); } while (count > 0); stop_time = rdtsc64(); ml_set_interrupts_enabled(state); return tsc_to_nanoseconds(stop_time - start_time); } static void rtc_lapic_timer_calibrate(void) { uint32_t nsecs; uint64_t countdown; if (!(cpuid_features() & CPUID_FEATURE_APIC)) return; /* * Set the local apic timer counting down to zero without an interrupt. * Use the timestamp to calculate how long this takes. */ nsecs = (uint32_t) lapic_time_countdown(rtc_intr_nsec); /* * Compute a countdown ratio for a given time in nanoseconds. * That is, countdown = time * numer / denom. */ countdown = (uint64_t)rtc_intr_nsec * (uint64_t)rtc_intr_nsec / nsecs; nsecs = (uint32_t) lapic_time_countdown((uint32_t) countdown); rtc_lapic_scale.numer = countdown; rtc_lapic_scale.denom = nsecs; kprintf("rtc_lapic_timer_calibrate() scale: %d/%d\n", (uint32_t) countdown, nsecs); } static void rtc_lapic_set_timer( uint32_t interval) { uint64_t count; assert(rtc_lapic_scale.denom); count = interval * (uint64_t) rtc_lapic_scale.numer; count /= rtc_lapic_scale.denom; lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count); } static void rtc_lapic_start_ticking(void) { uint64_t abstime; uint64_t first_tick; uint64_t decr; abstime = mach_absolute_time(); first_tick = abstime + NSEC_PER_HZ; current_cpu_datap()->cpu_rtc_tick_deadline = first_tick; decr = deadline_to_decrementer(first_tick, abstime); rtc_lapic_set_timer(decr); } /* * Configure the real-time clock device. Return success (1) * or failure (0). */ int sysclk_config(void) { mp_disable_preemption(); if (cpu_number() != master_cpu) { mp_enable_preemption(); return(1); } mp_enable_preemption(); timer_call_setup(&rtclock_alarm_timer, rtclock_alarm_expire, NULL); simple_lock_init(&rtclock.lock, 0); return (1); } /* * Nanotime/mach_absolutime_time * ----------------------------- * The timestamp counter (tsc) - which counts cpu clock cycles and can be read * efficient by the kernel and in userspace - is the reference for all timing. * However, the cpu clock rate is not only platform-dependent but can change * (speed-step) dynamically. Hence tsc is converted into nanoseconds which is * identical to mach_absolute_time. The conversion to tsc to nanoseconds is * encapsulated by nanotime. * * The kernel maintains nanotime information recording: * - the current ratio of tsc to nanoseconds * with this ratio expressed as a 32-bit scale and shift * (power of 2 divider); * - the tsc (step_tsc) and nanotime (step_ns) at which the current * ratio (clock speed) began. * So a tsc value can be converted to nanotime by: * * nanotime = (((tsc - step_tsc)*scale) >> shift) + step_ns * * In general, (tsc - step_tsc) is a 64-bit quantity with the scaling * involving a 96-bit intermediate value. However, by saving the converted * values at each tick (or at any intervening speed-step) - base_tsc and * base_ns - we can perform conversions relative to these and be assured that * (tsc - tick_tsc) is 32-bits. Hence: * * fast_nanotime = (((tsc - base_tsc)*scale) >> shift) + base_ns * * The tuple {base_tsc, base_ns, scale, shift} is exported in the commpage * for the userspace nanotime routine to read. A duplicate check_tsc is * appended so that the consistency of the read can be verified. Note that * this scheme is essential for MP systems in which the commpage is updated * by the master cpu but may be read concurrently by other cpus. * */ static inline void rtc_nanotime_set_commpage(rtc_nanotime_t *rntp) { commpage_nanotime_t cp_nanotime; /* Only the master cpu updates the commpage */ if (cpu_number() != master_cpu) return; cp_nanotime.nt_base_tsc = rntp->rnt_tsc; cp_nanotime.nt_base_ns = rntp->rnt_nanos; cp_nanotime.nt_scale = rntp->rnt_scale; cp_nanotime.nt_shift = rntp->rnt_shift; commpage_set_nanotime(&cp_nanotime); } static void rtc_nanotime_init(void) { rtc_nanotime_t *rntp = ¤t_cpu_datap()->cpu_rtc_nanotime; rtc_nanotime_t *master_rntp = &cpu_datap(master_cpu)->cpu_rtc_nanotime; if (cpu_number() == master_cpu) { rntp->rnt_tsc = rdtsc64(); rntp->rnt_nanos = tsc_to_nanoseconds(rntp->rnt_tsc); rntp->rnt_scale = rtc_quant_scale; rntp->rnt_shift = rtc_quant_shift; rntp->rnt_step_tsc = 0ULL; rntp->rnt_step_nanos = 0ULL; } else { /* * Copy master processor's nanotime info. * Loop required in case this changes while copying. */ do { *rntp = *master_rntp; } while (rntp->rnt_tsc != master_rntp->rnt_tsc); } } static inline void _rtc_nanotime_update(rtc_nanotime_t *rntp, uint64_t tsc) { uint64_t tsc_delta; uint64_t ns_delta; tsc_delta = tsc - rntp->rnt_step_tsc; ns_delta = tsc_to_nanoseconds(tsc_delta); rntp->rnt_nanos = rntp->rnt_step_nanos + ns_delta; rntp->rnt_tsc = tsc; } static void rtc_nanotime_update(void) { rtc_nanotime_t *rntp = ¤t_cpu_datap()->cpu_rtc_nanotime; assert(get_preemption_level() > 0); assert(!ml_get_interrupts_enabled()); _rtc_nanotime_update(rntp, rdtsc64()); rtc_nanotime_set_commpage(rntp); } static void rtc_nanotime_scale_update(void) { rtc_nanotime_t *rntp = ¤t_cpu_datap()->cpu_rtc_nanotime; uint64_t tsc = rdtsc64(); assert(!ml_get_interrupts_enabled()); /* * Update time based on past scale. */ _rtc_nanotime_update(rntp, tsc); /* * Update scale and timestamp this update. */ rntp->rnt_scale = rtc_quant_scale; rntp->rnt_shift = rtc_quant_shift; rntp->rnt_step_tsc = rntp->rnt_tsc; rntp->rnt_step_nanos = rntp->rnt_nanos; /* Export update to userland */ rtc_nanotime_set_commpage(rntp); } static uint64_t _rtc_nanotime_read(void) { rtc_nanotime_t *rntp = ¤t_cpu_datap()->cpu_rtc_nanotime; uint64_t rnt_tsc; uint32_t rnt_scale; uint32_t rnt_shift; uint64_t rnt_nanos; uint64_t tsc; uint64_t tsc_delta; rnt_scale = rntp->rnt_scale; if (rnt_scale == 0) return 0ULL; rnt_shift = rntp->rnt_shift; rnt_nanos = rntp->rnt_nanos; rnt_tsc = rntp->rnt_tsc; tsc = rdtsc64(); tsc_delta = tsc - rnt_tsc; if ((tsc_delta >> 32) != 0) return rnt_nanos + tsc_to_nanoseconds(tsc_delta); /* Let the compiler optimize(?): */ if (rnt_shift == 32) return rnt_nanos + ((tsc_delta * rnt_scale) >> 32); else return rnt_nanos + ((tsc_delta * rnt_scale) >> rnt_shift); } uint64_t rtc_nanotime_read(void) { uint64_t result; uint64_t rnt_tsc; rtc_nanotime_t *rntp = ¤t_cpu_datap()->cpu_rtc_nanotime; /* * Use timestamp to ensure the uptime record isn't changed. * This avoids disabling interrupts. * And not this is a per-cpu structure hence no locking. */ do { rnt_tsc = rntp->rnt_tsc; result = _rtc_nanotime_read(); } while (rnt_tsc != rntp->rnt_tsc); return result; } /* * This function is called by the speed-step driver when a * change of cpu clock frequency is about to occur. * The scale is not changed until rtc_clock_stepped() is called. * Between these times there is an uncertainty is exactly when * the change takes effect. FIXME: by using another timing source * we could eliminate this error. */ void rtc_clock_stepping(__unused uint32_t new_frequency, __unused uint32_t old_frequency) { boolean_t istate; istate = ml_set_interrupts_enabled(FALSE); rtc_nanotime_scale_update(); ml_set_interrupts_enabled(istate); } /* * This function is called by the speed-step driver when a * change of cpu clock frequency has just occured. This change * is expressed as a ratio relative to the boot clock rate. */ void rtc_clock_stepped(uint32_t new_frequency, uint32_t old_frequency) { boolean_t istate; istate = ml_set_interrupts_enabled(FALSE); if (rtc_boot_frequency == 0) { /* * At the first ever stepping, old frequency is the real * initial clock rate. This step and all others are based * relative to this initial frequency at which the tsc * calibration was made. Hence we must remember this base * frequency as reference. */ rtc_boot_frequency = old_frequency; } rtc_set_cyc_per_sec(rtc_cycle_count * new_frequency / rtc_boot_frequency); rtc_nanotime_scale_update(); ml_set_interrupts_enabled(istate); } /* * rtc_sleep_wakeup() is called from acpi on awakening from a S3 sleep */ void rtc_sleep_wakeup(void) { rtc_nanotime_t *rntp = ¤t_cpu_datap()->cpu_rtc_nanotime; boolean_t istate; istate = ml_set_interrupts_enabled(FALSE); /* * Reset nanotime. * The timestamp counter will have been reset * but nanotime (uptime) marches onward. * We assume that we're still at the former cpu frequency. */ rntp->rnt_tsc = rdtsc64(); rntp->rnt_step_tsc = 0ULL; rntp->rnt_step_nanos = rntp->rnt_nanos; rtc_nanotime_set_commpage(rntp); /* Restart tick interrupts from the LAPIC timer */ rtc_lapic_start_ticking(); ml_set_interrupts_enabled(istate); } /* * Initialize the real-time clock device. * In addition, various variables used to support the clock are initialized. */ int sysclk_init(void) { uint64_t cycles; mp_disable_preemption(); if (cpu_number() == master_cpu) { /* * Perform calibration. * The PIT is used as the reference to compute how many * TCS counts (cpu clock cycles) occur per second. */ rtc_cycle_count = timeRDTSC(); cycles = rtc_set_cyc_per_sec(rtc_cycle_count); /* * Set min/max to actual. * ACPI may update these later if speed-stepping is detected. */ gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles; gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles; printf("[RTCLOCK] frequency %llu (%llu)\n", cycles, rtc_cyc_per_sec); rtc_lapic_timer_calibrate(); /* Minimum interval is 1usec */ rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC, 0ULL); /* Point LAPIC interrupts to hardclock() */ lapic_set_timer_func((i386_intr_func_t) rtclock_intr); clock_timebase_init(); rtc_initialized = TRUE; } rtc_nanotime_init(); rtc_lapic_start_ticking(); mp_enable_preemption(); return (1); } /* * Get the clock device time. This routine is responsible * for converting the device's machine dependent time value * into a canonical mach_timespec_t value. */ static kern_return_t sysclk_gettime_internal( mach_timespec_t *cur_time) /* OUT */ { *cur_time = tsc_to_timespec(); return (KERN_SUCCESS); } kern_return_t sysclk_gettime( mach_timespec_t *cur_time) /* OUT */ { return sysclk_gettime_internal(cur_time); } void sysclk_gettime_interrupts_disabled( mach_timespec_t *cur_time) /* OUT */ { (void) sysclk_gettime_internal(cur_time); } // utility routine // Code to calculate how many processor cycles are in a second... static uint64_t rtc_set_cyc_per_sec(uint64_t cycles) { if (cycles > (NSEC_PER_SEC/20)) { // we can use just a "fast" multiply to get nanos rtc_quant_shift = 32; rtc_quant_scale = create_mul_quant_GHZ(rtc_quant_shift, cycles); rtclock.timebase_const.numer = rtc_quant_scale; // timeRDTSC is 1/20 rtclock.timebase_const.denom = RTC_FAST_DENOM; } else { rtc_quant_shift = 26; rtc_quant_scale = create_mul_quant_GHZ(rtc_quant_shift, cycles); rtclock.timebase_const.numer = NSEC_PER_SEC/20; // timeRDTSC is 1/20 rtclock.timebase_const.denom = cycles; } rtc_cyc_per_sec = cycles*20; // multiply it by 20 and we are done.. // BUT we also want to calculate... cycles = ((rtc_cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2)) / UI_CPUFREQ_ROUNDING_FACTOR) * UI_CPUFREQ_ROUNDING_FACTOR; /* * Set current measured speed. */ if (cycles >= 0x100000000ULL) { gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL; } else { gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles; } gPEClockFrequencyInfo.cpu_frequency_hz = cycles; kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, rtc_cyc_per_sec); return(cycles); } void clock_get_system_microtime( uint32_t *secs, uint32_t *microsecs) { mach_timespec_t now; (void) sysclk_gettime_internal(&now); *secs = now.tv_sec; *microsecs = now.tv_nsec / NSEC_PER_USEC; } void clock_get_system_nanotime( uint32_t *secs, uint32_t *nanosecs) { mach_timespec_t now; (void) sysclk_gettime_internal(&now); *secs = now.tv_sec; *nanosecs = now.tv_nsec; } /* * Get clock device attributes. */ kern_return_t sysclk_getattr( clock_flavor_t flavor, clock_attr_t attr, /* OUT */ mach_msg_type_number_t *count) /* IN/OUT */ { if (*count != 1) return (KERN_FAILURE); switch (flavor) { case CLOCK_GET_TIME_RES: /* >0 res */ *(clock_res_t *) attr = rtc_intr_nsec; break; case CLOCK_ALARM_CURRES: /* =0 no alarm */ case CLOCK_ALARM_MAXRES: case CLOCK_ALARM_MINRES: *(clock_res_t *) attr = 0; break; default: return (KERN_INVALID_VALUE); } return (KERN_SUCCESS); } /* * Set next alarm time for the clock device. This call * always resets the time to deliver an alarm for the * clock. */ void sysclk_setalarm( mach_timespec_t *alarm_time) { timer_call_enter(&rtclock_alarm_timer, (uint64_t) alarm_time->tv_sec * NSEC_PER_SEC + alarm_time->tv_nsec); } /* * Configure the calendar clock. */ int calend_config(void) { return bbc_config(); } /* * Initialize calendar clock. */ int calend_init(void) { return (1); } /* * Get the current clock time. */ kern_return_t calend_gettime( mach_timespec_t *cur_time) /* OUT */ { spl_t s; RTC_LOCK(s); if (!rtclock.calend_is_set) { RTC_UNLOCK(s); return (KERN_FAILURE); } (void) sysclk_gettime_internal(cur_time); ADD_MACH_TIMESPEC(cur_time, &rtclock.calend_offset); RTC_UNLOCK(s); return (KERN_SUCCESS); } void clock_get_calendar_microtime( uint32_t *secs, uint32_t *microsecs) { mach_timespec_t now; calend_gettime(&now); *secs = now.tv_sec; *microsecs = now.tv_nsec / NSEC_PER_USEC; } void clock_get_calendar_nanotime( uint32_t *secs, uint32_t *nanosecs) { mach_timespec_t now; calend_gettime(&now); *secs = now.tv_sec; *nanosecs = now.tv_nsec; } void clock_set_calendar_microtime( uint32_t secs, uint32_t microsecs) { mach_timespec_t new_time, curr_time; uint32_t old_offset; spl_t s; new_time.tv_sec = secs; new_time.tv_nsec = microsecs * NSEC_PER_USEC; RTC_LOCK(s); old_offset = rtclock.calend_offset.tv_sec; (void) sysclk_gettime_internal(&curr_time); rtclock.calend_offset = new_time; SUB_MACH_TIMESPEC(&rtclock.calend_offset, &curr_time); rtclock.boottime += rtclock.calend_offset.tv_sec - old_offset; rtclock.calend_is_set = TRUE; RTC_UNLOCK(s); (void) bbc_settime(&new_time); host_notify_calendar_change(); } /* * Get clock device attributes. */ kern_return_t calend_getattr( clock_flavor_t flavor, clock_attr_t attr, /* OUT */ mach_msg_type_number_t *count) /* IN/OUT */ { if (*count != 1) return (KERN_FAILURE); switch (flavor) { case CLOCK_GET_TIME_RES: /* >0 res */ *(clock_res_t *) attr = rtc_intr_nsec; break; case CLOCK_ALARM_CURRES: /* =0 no alarm */ case CLOCK_ALARM_MINRES: case CLOCK_ALARM_MAXRES: *(clock_res_t *) attr = 0; break; default: return (KERN_INVALID_VALUE); } return (KERN_SUCCESS); } #define tickadj (40*NSEC_PER_USEC) /* "standard" skew, ns / tick */ #define bigadj (NSEC_PER_SEC) /* use 10x skew above bigadj ns */ uint32_t clock_set_calendar_adjtime( int32_t *secs, int32_t *microsecs) { int64_t total, ototal; uint32_t interval = 0; spl_t s; total = (int64_t)*secs * NSEC_PER_SEC + *microsecs * NSEC_PER_USEC; RTC_LOCK(s); ototal = rtclock.calend_adjtotal; if (total != 0) { int32_t delta = tickadj; if (total > 0) { if (total > bigadj) delta *= 10; if (delta > total) delta = total; } else { if (total < -bigadj) delta *= 10; delta = -delta; if (delta < total) delta = total; } rtclock.calend_adjtotal = total; rtclock.calend_adjdelta = delta; interval = NSEC_PER_HZ; } else rtclock.calend_adjdelta = rtclock.calend_adjtotal = 0; RTC_UNLOCK(s); if (ototal == 0) *secs = *microsecs = 0; else { *secs = ototal / NSEC_PER_SEC; *microsecs = ototal % NSEC_PER_SEC; } return (interval); } uint32_t clock_adjust_calendar(void) { uint32_t interval = 0; int32_t delta; spl_t s; RTC_LOCK(s); delta = rtclock.calend_adjdelta; ADD_MACH_TIMESPEC_NSEC(&rtclock.calend_offset, delta); rtclock.calend_adjtotal -= delta; if (delta > 0) { if (delta > rtclock.calend_adjtotal) rtclock.calend_adjdelta = rtclock.calend_adjtotal; } else if (delta < 0) { if (delta < rtclock.calend_adjtotal) rtclock.calend_adjdelta = rtclock.calend_adjtotal; } if (rtclock.calend_adjdelta != 0) interval = NSEC_PER_HZ; RTC_UNLOCK(s); return (interval); } void clock_initialize_calendar(void) { mach_timespec_t bbc_time, curr_time; spl_t s; if (bbc_gettime(&bbc_time) != KERN_SUCCESS) return; RTC_LOCK(s); if (rtclock.boottime == 0) rtclock.boottime = bbc_time.tv_sec; (void) sysclk_gettime_internal(&curr_time); rtclock.calend_offset = bbc_time; SUB_MACH_TIMESPEC(&rtclock.calend_offset, &curr_time); rtclock.calend_is_set = TRUE; RTC_UNLOCK(s); host_notify_calendar_change(); } void clock_get_boottime_nanotime( uint32_t *secs, uint32_t *nanosecs) { *secs = rtclock.boottime; *nanosecs = 0; } void clock_timebase_info( mach_timebase_info_t info) { info->numer = info->denom = 1; } void clock_set_timer_deadline( uint64_t deadline) { spl_t s; cpu_data_t *pp = current_cpu_datap(); rtclock_timer_t *mytimer = &pp->cpu_rtc_timer; uint64_t abstime; uint64_t decr; assert(get_preemption_level() > 0); assert(rtclock_timer_expire); RTC_INTRS_OFF(s); mytimer->deadline = deadline; mytimer->is_set = TRUE; if (!mytimer->has_expired) { abstime = mach_absolute_time(); if (mytimer->deadline < pp->cpu_rtc_tick_deadline) { decr = deadline_to_decrementer(mytimer->deadline, abstime); rtc_lapic_set_timer(decr); pp->cpu_rtc_intr_deadline = mytimer->deadline; KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, decr, 2, 0, 0, 0); } } RTC_INTRS_ON(s); } void clock_set_timer_func( clock_timer_func_t func) { if (rtclock_timer_expire == NULL) rtclock_timer_expire = func; } /* * Real-time clock device interrupt. */ void rtclock_intr(struct i386_interrupt_state *regs) { uint64_t abstime; uint32_t latency; uint64_t decr; uint64_t decr_tick; uint64_t decr_timer; cpu_data_t *pp = current_cpu_datap(); rtclock_timer_t *mytimer = &pp->cpu_rtc_timer; assert(get_preemption_level() > 0); assert(!ml_get_interrupts_enabled()); abstime = _rtc_nanotime_read(); latency = (uint32_t) abstime - pp->cpu_rtc_intr_deadline; if (pp->cpu_rtc_tick_deadline <= abstime) { rtc_nanotime_update(); clock_deadline_for_periodic_event( NSEC_PER_HZ, abstime, &pp->cpu_rtc_tick_deadline); hertz_tick( #if STAT_TIME NSEC_PER_HZ, #endif (regs->efl & EFL_VM) || ((regs->cs & 0x03) != 0), regs->eip); } abstime = _rtc_nanotime_read(); if (mytimer->is_set && mytimer->deadline <= abstime) { mytimer->has_expired = TRUE; mytimer->is_set = FALSE; (*rtclock_timer_expire)(abstime); assert(!ml_get_interrupts_enabled()); mytimer->has_expired = FALSE; } /* Log the interrupt service latency (-ve value expected by tool) */ KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE, -latency, (uint32_t)regs->eip, 0, 0, 0); abstime = _rtc_nanotime_read(); decr_tick = deadline_to_decrementer(pp->cpu_rtc_tick_deadline, abstime); decr_timer = (mytimer->is_set) ? deadline_to_decrementer(mytimer->deadline, abstime) : DECREMENTER_MAX; decr = MIN(decr_tick, decr_timer); pp->cpu_rtc_intr_deadline = abstime + decr; rtc_lapic_set_timer(decr); /* Log the new decrementer value */ KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_EXCP_DECI, 1) | DBG_FUNC_NONE, decr, 3, 0, 0, 0); } static void rtclock_alarm_expire( __unused timer_call_param_t p0, __unused timer_call_param_t p1) { mach_timespec_t clock_time; (void) sysclk_gettime_internal(&clock_time); clock_alarm_intr(SYSTEM_CLOCK, &clock_time); } void clock_get_uptime( uint64_t *result) { *result = rtc_nanotime_read(); } uint64_t mach_absolute_time(void) { return rtc_nanotime_read(); } void absolutetime_to_microtime( uint64_t abstime, uint32_t *secs, uint32_t *microsecs) { uint32_t remain; asm volatile( "divl %3" : "=a" (*secs), "=d" (remain) : "A" (abstime), "r" (NSEC_PER_SEC)); asm volatile( "divl %3" : "=a" (*microsecs) : "0" (remain), "d" (0), "r" (NSEC_PER_USEC)); } void clock_interval_to_deadline( uint32_t interval, uint32_t scale_factor, uint64_t *result) { uint64_t abstime; clock_get_uptime(result); clock_interval_to_absolutetime_interval(interval, scale_factor, &abstime); *result += abstime; } void clock_interval_to_absolutetime_interval( uint32_t interval, uint32_t scale_factor, uint64_t *result) { *result = (uint64_t)interval * scale_factor; } void clock_absolutetime_interval_to_deadline( uint64_t abstime, uint64_t *result) { clock_get_uptime(result); *result += abstime; } void absolutetime_to_nanoseconds( uint64_t abstime, uint64_t *result) { *result = abstime; } void nanoseconds_to_absolutetime( uint64_t nanoseconds, uint64_t *result) { *result = nanoseconds; } void machine_delay_until( uint64_t deadline) { uint64_t now; do { cpu_pause(); now = mach_absolute_time(); } while (now < deadline); }