/* * Copyright (c) 1993-1995, 1999-2000 Apple Computer, Inc. * All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * The contents of this file constitute Original Code as defined in and * are subject to the Apple Public Source License Version 1.1 (the * "License"). You may not use this file except in compliance with the * License. Please obtain a copy of the License at * http://www.apple.com/publicsource and read it before using this file. * * This Original Code and all software distributed under the License are * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the * License for the specific language governing rights and limitations * under the License. * * @APPLE_LICENSE_HEADER_END@ */ /* * Thread-based callout module. * * HISTORY * * 10 July 1999 (debo) * Pulled into Mac OS X (microkernel). * * 3 July 1993 (debo) * Created. */ #include #include #include #include #include #include #include #include #include #define internal_call_num 768 #define thread_call_thread_min 4 static thread_call_data_t internal_call_storage[internal_call_num]; decl_simple_lock_data(static,thread_call_lock) static timer_call_data_t thread_call_delaytimer; static queue_head_t thread_call_xxx_queue, thread_call_pending_queue, thread_call_delayed_queue; static struct wait_queue call_thread_waitqueue; static boolean_t activate_thread_awake; static struct { int pending_num, pending_hiwat; int active_num, active_hiwat, active_lowat; int delayed_num, delayed_hiwat; int idle_thread_num; int thread_num, thread_hiwat, thread_lowat; } thread_call_vars; static __inline__ thread_call_t _internal_call_allocate(void); static __inline__ void _internal_call_release( thread_call_t call ); static __inline__ void _pending_call_enqueue( thread_call_t call ), _pending_call_dequeue( thread_call_t call ), _delayed_call_enqueue( thread_call_t call ), _delayed_call_dequeue( thread_call_t call ); static void __inline__ _set_delayed_call_timer( thread_call_t call ); static boolean_t _remove_from_pending_queue( thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all ), _remove_from_delayed_queue( thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all ); static __inline__ void _call_thread_wake(void); static void _call_thread(void), _activate_thread(void); static void _delayed_call_timer( timer_call_param_t p0, timer_call_param_t p1 ); #define qe(x) ((queue_entry_t)(x)) #define TC(x) ((thread_call_t)(x)) /* * Routine: thread_call_initialize [public] * * Description: Initialize this module, called * early during system initialization. * * Preconditions: None. * * Postconditions: None. */ void thread_call_initialize(void) { thread_call_t call; spl_t s; simple_lock_init(&thread_call_lock, ETAP_MISC_TIMER); s = splsched(); simple_lock(&thread_call_lock); queue_init(&thread_call_pending_queue); queue_init(&thread_call_delayed_queue); queue_init(&thread_call_xxx_queue); for ( call = internal_call_storage; call < &internal_call_storage[internal_call_num]; call++) { enqueue_tail(&thread_call_xxx_queue, qe(call)); } timer_call_setup(&thread_call_delaytimer, _delayed_call_timer, NULL); wait_queue_init(&call_thread_waitqueue, SYNC_POLICY_FIFO); thread_call_vars.thread_lowat = thread_call_thread_min; activate_thread_awake = TRUE; simple_unlock(&thread_call_lock); splx(s); kernel_thread_with_priority(_activate_thread, MAXPRI_KERNEL - 2); } void thread_call_setup( thread_call_t call, thread_call_func_t func, thread_call_param_t param0 ) { call_entry_setup(call, func, param0); } /* * Routine: _internal_call_allocate [private, inline] * * Purpose: Allocate an internal callout entry. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ thread_call_t _internal_call_allocate(void) { thread_call_t call; if (queue_empty(&thread_call_xxx_queue)) panic("_internal_call_allocate"); call = TC(dequeue_head(&thread_call_xxx_queue)); return (call); } /* * Routine: _internal_call_release [private, inline] * * Purpose: Release an internal callout entry which * is no longer pending (or delayed). * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _internal_call_release( thread_call_t call ) { if ( call >= internal_call_storage && call < &internal_call_storage[internal_call_num] ) enqueue_head(&thread_call_xxx_queue, qe(call)); } /* * Routine: _pending_call_enqueue [private, inline] * * Purpose: Place an entry at the end of the * pending queue, to be executed soon. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _pending_call_enqueue( thread_call_t call ) { enqueue_tail(&thread_call_pending_queue, qe(call)); if (++thread_call_vars.pending_num > thread_call_vars.pending_hiwat) thread_call_vars.pending_hiwat = thread_call_vars.pending_num; call->state = PENDING; } /* * Routine: _pending_call_dequeue [private, inline] * * Purpose: Remove an entry from the pending queue, * effectively unscheduling it. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _pending_call_dequeue( thread_call_t call ) { (void)remque(qe(call)); thread_call_vars.pending_num--; call->state = IDLE; } /* * Routine: _delayed_call_enqueue [private, inline] * * Purpose: Place an entry on the delayed queue, * after existing entries with an earlier * (or identical) deadline. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _delayed_call_enqueue( thread_call_t call ) { thread_call_t current; current = TC(queue_first(&thread_call_delayed_queue)); while (TRUE) { if ( queue_end(&thread_call_delayed_queue, qe(current)) || call->deadline < current->deadline ) { current = TC(queue_prev(qe(current))); break; } current = TC(queue_next(qe(current))); } insque(qe(call), qe(current)); if (++thread_call_vars.delayed_num > thread_call_vars.delayed_hiwat) thread_call_vars.delayed_hiwat = thread_call_vars.delayed_num; call->state = DELAYED; } /* * Routine: _delayed_call_dequeue [private, inline] * * Purpose: Remove an entry from the delayed queue, * effectively unscheduling it. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _delayed_call_dequeue( thread_call_t call ) { (void)remque(qe(call)); thread_call_vars.delayed_num--; call->state = IDLE; } /* * Routine: _set_delayed_call_timer [private] * * Purpose: Reset the timer so that it * next expires when the entry is due. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _set_delayed_call_timer( thread_call_t call ) { timer_call_enter(&thread_call_delaytimer, call->deadline); } /* * Routine: _remove_from_pending_queue [private] * * Purpose: Remove the first (or all) matching * entries from the pending queue, * effectively unscheduling them. * Returns whether any matching entries * were found. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static boolean_t _remove_from_pending_queue( thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all ) { boolean_t call_removed = FALSE; thread_call_t call; call = TC(queue_first(&thread_call_pending_queue)); while (!queue_end(&thread_call_pending_queue, qe(call))) { if ( call->func == func && call->param0 == param0 ) { thread_call_t next = TC(queue_next(qe(call))); _pending_call_dequeue(call); _internal_call_release(call); call_removed = TRUE; if (!remove_all) break; call = next; } else call = TC(queue_next(qe(call))); } return (call_removed); } /* * Routine: _remove_from_delayed_queue [private] * * Purpose: Remove the first (or all) matching * entries from the delayed queue, * effectively unscheduling them. * Returns whether any matching entries * were found. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static boolean_t _remove_from_delayed_queue( thread_call_func_t func, thread_call_param_t param0, boolean_t remove_all ) { boolean_t call_removed = FALSE; thread_call_t call; call = TC(queue_first(&thread_call_delayed_queue)); while (!queue_end(&thread_call_delayed_queue, qe(call))) { if ( call->func == func && call->param0 == param0 ) { thread_call_t next = TC(queue_next(qe(call))); _delayed_call_dequeue(call); _internal_call_release(call); call_removed = TRUE; if (!remove_all) break; call = next; } else call = TC(queue_next(qe(call))); } return (call_removed); } /* * Routine: thread_call_func [public] * * Purpose: Schedule a function callout. * Guarantees { function, argument } * uniqueness if unique_call is TRUE. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ void thread_call_func( thread_call_func_t func, thread_call_param_t param, boolean_t unique_call ) { thread_call_t call; spl_t s; s = splsched(); simple_lock(&thread_call_lock); call = TC(queue_first(&thread_call_pending_queue)); while (unique_call && !queue_end(&thread_call_pending_queue, qe(call))) { if ( call->func == func && call->param0 == param ) { break; } call = TC(queue_next(qe(call))); } if (!unique_call || queue_end(&thread_call_pending_queue, qe(call))) { call = _internal_call_allocate(); call->func = func; call->param0 = param; call->param1 = 0; _pending_call_enqueue(call); if (thread_call_vars.active_num <= 0) _call_thread_wake(); } simple_unlock(&thread_call_lock); splx(s); } /* * Routine: thread_call_func_delayed [public] * * Purpose: Schedule a function callout to * occur at the stated time. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ void thread_call_func_delayed( thread_call_func_t func, thread_call_param_t param, uint64_t deadline ) { thread_call_t call; spl_t s; s = splsched(); simple_lock(&thread_call_lock); call = _internal_call_allocate(); call->func = func; call->param0 = param; call->param1 = 0; call->deadline = deadline; _delayed_call_enqueue(call); if (queue_first(&thread_call_delayed_queue) == qe(call)) _set_delayed_call_timer(call); simple_unlock(&thread_call_lock); splx(s); } /* * Routine: thread_call_func_cancel [public] * * Purpose: Unschedule a function callout. * Removes one (or all) * { function, argument } * instance(s) from either (or both) * the pending and the delayed queue, * in that order. Returns a boolean * indicating whether any calls were * cancelled. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ boolean_t thread_call_func_cancel( thread_call_func_t func, thread_call_param_t param, boolean_t cancel_all ) { boolean_t result; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (cancel_all) result = _remove_from_pending_queue(func, param, cancel_all) | _remove_from_delayed_queue(func, param, cancel_all); else result = _remove_from_pending_queue(func, param, cancel_all) || _remove_from_delayed_queue(func, param, cancel_all); simple_unlock(&thread_call_lock); splx(s); return (result); } /* * Routine: thread_call_allocate [public] * * Purpose: Allocate an external callout * entry. * * Preconditions: None. * * Postconditions: None. */ thread_call_t thread_call_allocate( thread_call_func_t func, thread_call_param_t param0 ) { thread_call_t call = (void *)kalloc(sizeof (thread_call_data_t)); call->func = func; call->param0 = param0; call->state = IDLE; return (call); } /* * Routine: thread_call_free [public] * * Purpose: Free an external callout * entry. * * Preconditions: None. * * Postconditions: None. */ boolean_t thread_call_free( thread_call_t call ) { spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state != IDLE) { simple_unlock(&thread_call_lock); splx(s); return (FALSE); } simple_unlock(&thread_call_lock); splx(s); kfree((vm_offset_t)call, sizeof (thread_call_data_t)); return (TRUE); } /* * Routine: thread_call_enter [public] * * Purpose: Schedule an external callout * entry to occur "soon". Returns a * boolean indicating whether the call * had been already scheduled. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ boolean_t thread_call_enter( thread_call_t call ) { boolean_t result = TRUE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state != PENDING) { if (call->state == DELAYED) _delayed_call_dequeue(call); else if (call->state == IDLE) result = FALSE; _pending_call_enqueue(call); if (thread_call_vars.active_num <= 0) _call_thread_wake(); } call->param1 = 0; simple_unlock(&thread_call_lock); splx(s); return (result); } boolean_t thread_call_enter1( thread_call_t call, thread_call_param_t param1 ) { boolean_t result = TRUE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state != PENDING) { if (call->state == DELAYED) _delayed_call_dequeue(call); else if (call->state == IDLE) result = FALSE; _pending_call_enqueue(call); if (thread_call_vars.active_num <= 0) _call_thread_wake(); } call->param1 = param1; simple_unlock(&thread_call_lock); splx(s); return (result); } /* * Routine: thread_call_enter_delayed [public] * * Purpose: Schedule an external callout * entry to occur at the stated time. * Returns a boolean indicating whether * the call had been already scheduled. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ boolean_t thread_call_enter_delayed( thread_call_t call, uint64_t deadline ) { boolean_t result = TRUE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state == PENDING) _pending_call_dequeue(call); else if (call->state == DELAYED) _delayed_call_dequeue(call); else if (call->state == IDLE) result = FALSE; call->param1 = 0; call->deadline = deadline; _delayed_call_enqueue(call); if (queue_first(&thread_call_delayed_queue) == qe(call)) _set_delayed_call_timer(call); simple_unlock(&thread_call_lock); splx(s); return (result); } boolean_t thread_call_enter1_delayed( thread_call_t call, thread_call_param_t param1, uint64_t deadline ) { boolean_t result = TRUE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state == PENDING) _pending_call_dequeue(call); else if (call->state == DELAYED) _delayed_call_dequeue(call); else if (call->state == IDLE) result = FALSE; call->param1 = param1; call->deadline = deadline; _delayed_call_enqueue(call); if (queue_first(&thread_call_delayed_queue) == qe(call)) _set_delayed_call_timer(call); simple_unlock(&thread_call_lock); splx(s); return (result); } /* * Routine: thread_call_cancel [public] * * Purpose: Unschedule a callout entry. * Returns a boolean indicating * whether the call had actually * been scheduled. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ boolean_t thread_call_cancel( thread_call_t call ) { boolean_t result = TRUE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state == PENDING) _pending_call_dequeue(call); else if (call->state == DELAYED) _delayed_call_dequeue(call); else result = FALSE; simple_unlock(&thread_call_lock); splx(s); return (result); } /* * Routine: thread_call_is_delayed [public] * * Purpose: Returns a boolean indicating * whether a call is currently scheduled * to occur at a later time. Optionally * returns the expiration time. * * Preconditions: Callable from an interrupt context * below splsched. * * Postconditions: None. */ boolean_t thread_call_is_delayed( thread_call_t call, uint64_t *deadline) { boolean_t result = FALSE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); if (call->state == DELAYED) { if (deadline != NULL) *deadline = call->deadline; result = TRUE; } simple_unlock(&thread_call_lock); splx(s); return (result); } /* * Routine: _call_thread_wake [private, inline] * * Purpose: Wake a callout thread to service * pending callout entries. May wake * the activate thread in order to * create additional callout threads. * * Preconditions: thread_call_lock held. * * Postconditions: None. */ static __inline__ void _call_thread_wake(void) { if (wait_queue_wakeup_one( &call_thread_waitqueue, &call_thread_waitqueue, THREAD_AWAKENED) == KERN_SUCCESS) { thread_call_vars.idle_thread_num--; if (++thread_call_vars.active_num > thread_call_vars.active_hiwat) thread_call_vars.active_hiwat = thread_call_vars.active_num; } else if (!activate_thread_awake) { thread_wakeup_one(&activate_thread_awake); activate_thread_awake = TRUE; } } /* * Routine: call_thread_block [private] * * Purpose: Hook via thread dispatch on * the occasion of a callout blocking. * * Preconditions: splsched. * * Postconditions: None. */ void call_thread_block(void) { simple_lock(&thread_call_lock); if (--thread_call_vars.active_num < thread_call_vars.active_lowat) thread_call_vars.active_lowat = thread_call_vars.active_num; if ( thread_call_vars.active_num <= 0 && thread_call_vars.pending_num > 0 ) _call_thread_wake(); simple_unlock(&thread_call_lock); } /* * Routine: call_thread_unblock [private] * * Purpose: Hook via thread wakeup on * the occasion of a callout unblocking. * * Preconditions: splsched. * * Postconditions: None. */ void call_thread_unblock(void) { simple_lock(&thread_call_lock); if (++thread_call_vars.active_num > thread_call_vars.active_hiwat) thread_call_vars.active_hiwat = thread_call_vars.active_num; simple_unlock(&thread_call_lock); } /* * Routine: _call_thread [private] * * Purpose: Executed by a callout thread. * * Preconditions: None. * * Postconditions: None. */ static void _call_thread_continue(void) { thread_t self = current_thread(); (void) splsched(); simple_lock(&thread_call_lock); self->active_callout = TRUE; while (thread_call_vars.pending_num > 0) { thread_call_t call; thread_call_func_t func; thread_call_param_t param0, param1; call = TC(dequeue_head(&thread_call_pending_queue)); thread_call_vars.pending_num--; func = call->func; param0 = call->param0; param1 = call->param1; call->state = IDLE; _internal_call_release(call); simple_unlock(&thread_call_lock); (void) spllo(); KERNEL_DEBUG_CONSTANT( MACHDBG_CODE(DBG_MACH_SCHED,MACH_CALLOUT) | DBG_FUNC_NONE, (int)func, (int)param0, (int)param1, 0, 0); (*func)(param0, param1); (void)thread_funnel_set(self->funnel_lock, FALSE); (void) splsched(); simple_lock(&thread_call_lock); } self->active_callout = FALSE; if (--thread_call_vars.active_num < thread_call_vars.active_lowat) thread_call_vars.active_lowat = thread_call_vars.active_num; if (thread_call_vars.idle_thread_num < thread_call_vars.thread_lowat) { thread_call_vars.idle_thread_num++; wait_queue_assert_wait( &call_thread_waitqueue, &call_thread_waitqueue, THREAD_INTERRUPTIBLE); simple_unlock(&thread_call_lock); (void) spllo(); thread_block(_call_thread_continue); /* NOTREACHED */ } thread_call_vars.thread_num--; simple_unlock(&thread_call_lock); (void) spllo(); (void) thread_terminate(self->top_act); /* NOTREACHED */ } static void _call_thread(void) { _call_thread_continue(); /* NOTREACHED */ } /* * Routine: _activate_thread [private] * * Purpose: Executed by the activate thread. * * Preconditions: None. * * Postconditions: Never terminates. */ static void _activate_thread_continue(void) { (void) splsched(); simple_lock(&thread_call_lock); while ( thread_call_vars.active_num <= 0 && thread_call_vars.pending_num > 0 ) { if (++thread_call_vars.active_num > thread_call_vars.active_hiwat) thread_call_vars.active_hiwat = thread_call_vars.active_num; if (++thread_call_vars.thread_num > thread_call_vars.thread_hiwat) thread_call_vars.thread_hiwat = thread_call_vars.thread_num; simple_unlock(&thread_call_lock); (void) spllo(); kernel_thread_with_priority(_call_thread, MAXPRI_KERNEL - 1); (void) splsched(); simple_lock(&thread_call_lock); } assert_wait(&activate_thread_awake, THREAD_INTERRUPTIBLE); activate_thread_awake = FALSE; simple_unlock(&thread_call_lock); (void) spllo(); thread_block(_activate_thread_continue); /* NOTREACHED */ } static void _activate_thread(void) { thread_t self = current_thread(); self->vm_privilege = TRUE; vm_page_free_reserve(2); /* XXX */ _activate_thread_continue(); /* NOTREACHED */ } static void _delayed_call_timer( timer_call_param_t p0, timer_call_param_t p1 ) { uint64_t timestamp; thread_call_t call; boolean_t new_pending = FALSE; spl_t s; s = splsched(); simple_lock(&thread_call_lock); clock_get_uptime(×tamp); call = TC(queue_first(&thread_call_delayed_queue)); while (!queue_end(&thread_call_delayed_queue, qe(call))) { if (call->deadline <= timestamp) { _delayed_call_dequeue(call); _pending_call_enqueue(call); new_pending = TRUE; } else break; call = TC(queue_first(&thread_call_delayed_queue)); } if (!queue_end(&thread_call_delayed_queue, qe(call))) _set_delayed_call_timer(call); if (new_pending && thread_call_vars.active_num <= 0) _call_thread_wake(); simple_unlock(&thread_call_lock); splx(s); }