/* * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the * License may not be used to create, or enable the creation or * redistribution of, unlawful or unlicensed copies of an Apple operating * system, or to circumvent, violate, or enable the circumvention or * violation of, any terms of an Apple operating system software license * agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this * file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * File: i386/rtclock.c * Purpose: Routines for handling the machine dependent * real-time clock. Historically, this clock is * generated by the Intel 8254 Programmable Interval * Timer, but local apic timers are now used for * this purpose with the master time reference being * the cpu clock counted by the timestamp MSR. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for kernel_map */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MAX(a,b) (((a)>(b))?(a):(b)) #define MIN(a,b) (((a)>(b))?(b):(a)) #define NSEC_PER_HZ (NSEC_PER_SEC / 100) /* nsec per tick */ #define UI_CPUFREQ_ROUNDING_FACTOR 10000000 int rtclock_config(void); int rtclock_init(void); uint64_t rtc_decrementer_min; void rtclock_intr(x86_saved_state_t *regs); static uint64_t maxDec; /* longest interval our hardware timer can handle (nsec) */ /* XXX this should really be in a header somewhere */ extern clock_timer_func_t rtclock_timer_expire; static void rtc_set_timescale(uint64_t cycles); static uint64_t rtc_export_speed(uint64_t cycles); extern void rtc_nanotime_store( uint64_t tsc, uint64_t nsec, uint32_t scale, uint32_t shift, rtc_nanotime_t *dst); extern void rtc_nanotime_load( rtc_nanotime_t *src, rtc_nanotime_t *dst); rtc_nanotime_t rtc_nanotime_info; /* * tsc_to_nanoseconds: * * Basic routine to convert a raw 64 bit TSC value to a * 64 bit nanosecond value. The conversion is implemented * based on the scale factor and an implicit 32 bit shift. */ static inline uint64_t _tsc_to_nanoseconds(uint64_t value) { asm volatile("movl %%edx,%%esi ;" "mull %%ecx ;" "movl %%edx,%%edi ;" "movl %%esi,%%eax ;" "mull %%ecx ;" "addl %%edi,%%eax ;" "adcl $0,%%edx " : "+A" (value) : "c" (rtc_nanotime_info.scale) : "esi", "edi"); return (value); } uint64_t tsc_to_nanoseconds(uint64_t value) { return _tsc_to_nanoseconds(value); } static uint32_t deadline_to_decrementer( uint64_t deadline, uint64_t now) { uint64_t delta; if (deadline <= now) return rtc_decrementer_min; else { delta = deadline - now; return MIN(MAX(rtc_decrementer_min,delta),maxDec); } } static void rtc_lapic_start_ticking(void) { uint64_t abstime; uint64_t first_tick; cpu_data_t *cdp = current_cpu_datap(); abstime = mach_absolute_time(); rtclock_tick_interval = NSEC_PER_HZ; first_tick = abstime + rtclock_tick_interval; cdp->rtclock_intr_deadline = first_tick; /* * Force a complete re-evaluation of timer deadlines. */ cdp->rtcPop = EndOfAllTime; etimer_resync_deadlines(); } /* * Configure the real-time clock device. Return success (1) * or failure (0). */ int rtclock_config(void) { /* nothing to do */ return (1); } /* * Nanotime/mach_absolutime_time * ----------------------------- * The timestamp counter (TSC) - which counts cpu clock cycles and can be read * efficiently by the kernel and in userspace - is the reference for all timing. * The cpu clock rate is platform-dependent and may stop or be reset when the * processor is napped/slept. As a result, nanotime is the software abstraction * used to maintain a monotonic clock, adjusted from an outside reference as needed. * * The kernel maintains nanotime information recording: * - the ratio of tsc to nanoseconds * with this ratio expressed as a 32-bit scale and shift * (power of 2 divider); * - { tsc_base, ns_base } pair of corresponding timestamps. * * The tuple {tsc_base, ns_base, scale, shift} is exported in the commpage * for the userspace nanotime routine to read. * * All of the routines which update the nanotime data are non-reentrant. This must * be guaranteed by the caller. */ static inline void rtc_nanotime_set_commpage(rtc_nanotime_t *rntp) { commpage_set_nanotime(rntp->tsc_base, rntp->ns_base, rntp->scale, rntp->shift); } /* * rtc_nanotime_init: * * Intialize the nanotime info from the base time. Since * the base value might be from a lower resolution clock, * we compare it to the TSC derived value, and use the * greater of the two values. */ static inline void _rtc_nanotime_init(rtc_nanotime_t *rntp, uint64_t base) { uint64_t nsecs, tsc = rdtsc64(); nsecs = _tsc_to_nanoseconds(tsc); rtc_nanotime_store(tsc, MAX(nsecs, base), rntp->scale, rntp->shift, rntp); } static void rtc_nanotime_init(uint64_t base) { rtc_nanotime_t *rntp = &rtc_nanotime_info; _rtc_nanotime_init(rntp, base); rtc_nanotime_set_commpage(rntp); } /* * rtc_nanotime_init: * * Call back from the commpage initialization to * cause the commpage data to be filled in once the * commpages have been created. */ void rtc_nanotime_init_commpage(void) { spl_t s = splclock(); rtc_nanotime_set_commpage(&rtc_nanotime_info); splx(s); } /* * rtc_nanotime_update: * * Update the nanotime info from the base time. Since * the base value might be from a lower resolution clock, * we compare it to the TSC derived value, and use the * greater of the two values. * * N.B. In comparison to the above init routine, this assumes * that the TSC has remained monotonic compared to the tsc_base * value, which is not the case after S3 sleep. */ static inline void _rtc_nanotime_update(rtc_nanotime_t *rntp, uint64_t base) { uint64_t nsecs, tsc = rdtsc64(); nsecs = rntp->ns_base + _tsc_to_nanoseconds(tsc - rntp->tsc_base); rtc_nanotime_store(tsc, MAX(nsecs, base), rntp->scale, rntp->shift, rntp); } static void rtc_nanotime_update( uint64_t base) { rtc_nanotime_t *rntp = &rtc_nanotime_info; assert(!ml_get_interrupts_enabled()); _rtc_nanotime_update(rntp, base); rtc_nanotime_set_commpage(rntp); } /* * rtc_nanotime_read: * * Returns the current nanotime value, accessable from any * context. */ static uint64_t rtc_nanotime_read(void) { rtc_nanotime_t rnt, *rntp = &rtc_nanotime_info; uint64_t result; do { rtc_nanotime_load(rntp, &rnt); result = rnt.ns_base + _tsc_to_nanoseconds(rdtsc64() - rnt.tsc_base); } while (rntp->tsc_base != rnt.tsc_base); return (result); } /* * rtc_clock_napped: * * Invoked from power manangement when we have awoken from a nap (C3/C4) * during which the TSC lost counts. The nanotime data is updated according * to the provided nanosecond base value. * * The caller must guarantee non-reentrancy. */ void rtc_clock_napped( uint64_t base) { rtc_nanotime_update(base); } void rtc_clock_stepping(__unused uint32_t new_frequency, __unused uint32_t old_frequency) { panic("rtc_clock_stepping unsupported"); } void rtc_clock_stepped(__unused uint32_t new_frequency, __unused uint32_t old_frequency) { panic("rtc_clock_stepping unsupported"); } /* * rtc_sleep_wakeup: * * Invoked from power manageent when we have awoken from a sleep (S3) * and the TSC has been reset. The nanotime data is updated based on * the HPET value. * * The caller must guarantee non-reentrancy. */ void rtc_sleep_wakeup(void) { boolean_t istate; istate = ml_set_interrupts_enabled(FALSE); /* * Reset nanotime. * The timestamp counter will have been reset * but nanotime (uptime) marches onward. */ rtc_nanotime_init(tmrCvt(rdHPET(), hpetCvtt2n)); /* Restart tick interrupts from the LAPIC timer */ rtc_lapic_start_ticking(); ml_set_interrupts_enabled(istate); } /* * Initialize the real-time clock device. * In addition, various variables used to support the clock are initialized. */ int rtclock_init(void) { uint64_t cycles; assert(!ml_get_interrupts_enabled()); if (cpu_number() == master_cpu) { assert(tscFreq); rtc_set_timescale(tscFreq); /* * Adjust and set the exported cpu speed. */ cycles = rtc_export_speed(tscFreq); /* * Set min/max to actual. * ACPI may update these later if speed-stepping is detected. */ gPEClockFrequencyInfo.cpu_frequency_min_hz = cycles; gPEClockFrequencyInfo.cpu_frequency_max_hz = cycles; /* * Compute the longest interval we can represent. */ maxDec = tmrCvt(0x7fffffffULL, busFCvtt2n); kprintf("maxDec: %lld\n", maxDec); /* Minimum interval is 1usec */ rtc_decrementer_min = deadline_to_decrementer(NSEC_PER_USEC, 0ULL); /* Point LAPIC interrupts to hardclock() */ lapic_set_timer_func((i386_intr_func_t) rtclock_intr); clock_timebase_init(); ml_init_lock_timeout(); } rtc_lapic_start_ticking(); return (1); } // utility routine // Code to calculate how many processor cycles are in a second... static void rtc_set_timescale(uint64_t cycles) { rtc_nanotime_info.scale = ((uint64_t)NSEC_PER_SEC << 32) / cycles; rtc_nanotime_info.shift = 32; rtc_nanotime_init(0); } static uint64_t rtc_export_speed(uint64_t cyc_per_sec) { uint64_t cycles; /* Round: */ cycles = ((cyc_per_sec + (UI_CPUFREQ_ROUNDING_FACTOR/2)) / UI_CPUFREQ_ROUNDING_FACTOR) * UI_CPUFREQ_ROUNDING_FACTOR; /* * Set current measured speed. */ if (cycles >= 0x100000000ULL) { gPEClockFrequencyInfo.cpu_clock_rate_hz = 0xFFFFFFFFUL; } else { gPEClockFrequencyInfo.cpu_clock_rate_hz = (unsigned long)cycles; } gPEClockFrequencyInfo.cpu_frequency_hz = cycles; kprintf("[RTCLOCK] frequency %llu (%llu)\n", cycles, cyc_per_sec); return(cycles); } void clock_get_system_microtime( uint32_t *secs, uint32_t *microsecs) { uint64_t now = rtc_nanotime_read(); uint32_t remain; asm volatile( "divl %3" : "=a" (*secs), "=d" (remain) : "A" (now), "r" (NSEC_PER_SEC)); asm volatile( "divl %3" : "=a" (*microsecs) : "0" (remain), "d" (0), "r" (NSEC_PER_USEC)); } void clock_get_system_nanotime( uint32_t *secs, uint32_t *nanosecs) { uint64_t now = rtc_nanotime_read(); asm volatile( "divl %3" : "=a" (*secs), "=d" (*nanosecs) : "A" (now), "r" (NSEC_PER_SEC)); } void clock_gettimeofday_set_commpage( uint64_t abstime, uint64_t epoch, uint64_t offset, uint32_t *secs, uint32_t *microsecs) { uint64_t now = abstime; uint32_t remain; now += offset; asm volatile( "divl %3" : "=a" (*secs), "=d" (remain) : "A" (now), "r" (NSEC_PER_SEC)); asm volatile( "divl %3" : "=a" (*microsecs) : "0" (remain), "d" (0), "r" (NSEC_PER_USEC)); *secs += epoch; commpage_set_timestamp(abstime - remain, *secs, NSEC_PER_SEC); } void clock_timebase_info( mach_timebase_info_t info) { info->numer = info->denom = 1; } void clock_set_timer_func( clock_timer_func_t func) { if (rtclock_timer_expire == NULL) rtclock_timer_expire = func; } /* * Real-time clock device interrupt. */ void rtclock_intr( x86_saved_state_t *tregs) { uint64_t rip; boolean_t user_mode = FALSE; uint64_t abstime; uint32_t latency; cpu_data_t *pp = current_cpu_datap(); assert(get_preemption_level() > 0); assert(!ml_get_interrupts_enabled()); abstime = rtc_nanotime_read(); latency = (uint32_t) abstime - pp->rtcPop; if (is_saved_state64(tregs) == TRUE) { x86_saved_state64_t *regs; regs = saved_state64(tregs); user_mode = TRUE; rip = regs->isf.rip; } else { x86_saved_state32_t *regs; regs = saved_state32(tregs); if (regs->cs & 0x03) user_mode = TRUE; rip = regs->eip; } /* Log the interrupt service latency (-ve value expected by tool) */ KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_EXCP_DECI, 0) | DBG_FUNC_NONE, -latency, (uint32_t)rip, user_mode, 0, 0); /* call the generic etimer */ etimer_intr(user_mode, rip); } /* * Request timer pop from the hardware */ int setPop( uint64_t time) { uint64_t now; uint32_t decr; uint64_t count; now = rtc_nanotime_read(); /* The time in nanoseconds */ decr = deadline_to_decrementer(time, now); count = tmrCvt(decr, busFCvtn2t); lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t) count); return decr; /* Pass back what we set */ } void resetPop(void) { uint64_t now; uint32_t decr; uint64_t count; cpu_data_t *cdp = current_cpu_datap(); now = rtc_nanotime_read(); decr = deadline_to_decrementer(cdp->rtcPop, now); count = tmrCvt(decr, busFCvtn2t); lapic_set_timer(TRUE, one_shot, divide_by_1, (uint32_t)count); } uint64_t mach_absolute_time(void) { return rtc_nanotime_read(); } void clock_interval_to_absolutetime_interval( uint32_t interval, uint32_t scale_factor, uint64_t *result) { *result = (uint64_t)interval * scale_factor; } void absolutetime_to_microtime( uint64_t abstime, uint32_t *secs, uint32_t *microsecs) { uint32_t remain; asm volatile( "divl %3" : "=a" (*secs), "=d" (remain) : "A" (abstime), "r" (NSEC_PER_SEC)); asm volatile( "divl %3" : "=a" (*microsecs) : "0" (remain), "d" (0), "r" (NSEC_PER_USEC)); } void absolutetime_to_nanotime( uint64_t abstime, uint32_t *secs, uint32_t *nanosecs) { asm volatile( "divl %3" : "=a" (*secs), "=d" (*nanosecs) : "A" (abstime), "r" (NSEC_PER_SEC)); } void nanotime_to_absolutetime( uint32_t secs, uint32_t nanosecs, uint64_t *result) { *result = ((uint64_t)secs * NSEC_PER_SEC) + nanosecs; } void absolutetime_to_nanoseconds( uint64_t abstime, uint64_t *result) { *result = abstime; } void nanoseconds_to_absolutetime( uint64_t nanoseconds, uint64_t *result) { *result = nanoseconds; } void machine_delay_until( uint64_t deadline) { uint64_t now; do { cpu_pause(); now = mach_absolute_time(); } while (now < deadline); }