/* * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * * The contents of this file constitute Original Code as defined in and * are subject to the Apple Public Source License Version 1.1 (the * "License"). You may not use this file except in compliance with the * License. Please obtain a copy of the License at * http://www.apple.com/publicsource and read it before using this file. * * This Original Code and all software distributed under the License are * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the * License for the specific language governing rights and limitations * under the License. * * @APPLE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ */ /* * Mach Operating System * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University * All Rights Reserved. * * Permission to use, copy, modify and distribute this software and its * documentation is hereby granted, provided that both the copyright * notice and this permission notice appear in all copies of the * software, derivative works or modified versions, and any portions * thereof, and that both notices appear in supporting documentation. * * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. * * Carnegie Mellon requests users of this software to return to * * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU * School of Computer Science * Carnegie Mellon University * Pittsburgh PA 15213-3890 * * any improvements or extensions that they make and grant Carnegie Mellon * the rights to redistribute these changes. */ /* */ /* * File: vm/vm_page.c * Author: Avadis Tevanian, Jr., Michael Wayne Young * * Resident memory management module. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* kernel_memory_allocate() */ #include #include #include /* * Associated with page of user-allocatable memory is a * page structure. */ /* * These variables record the values returned by vm_page_bootstrap, * for debugging purposes. The implementation of pmap_steal_memory * and pmap_startup here also uses them internally. */ vm_offset_t virtual_space_start; vm_offset_t virtual_space_end; int vm_page_pages; /* * The vm_page_lookup() routine, which provides for fast * (virtual memory object, offset) to page lookup, employs * the following hash table. The vm_page_{insert,remove} * routines install and remove associations in the table. * [This table is often called the virtual-to-physical, * or VP, table.] */ typedef struct { vm_page_t pages; #if MACH_PAGE_HASH_STATS int cur_count; /* current count */ int hi_count; /* high water mark */ #endif /* MACH_PAGE_HASH_STATS */ } vm_page_bucket_t; vm_page_bucket_t *vm_page_buckets; /* Array of buckets */ unsigned int vm_page_bucket_count = 0; /* How big is array? */ unsigned int vm_page_hash_mask; /* Mask for hash function */ unsigned int vm_page_hash_shift; /* Shift for hash function */ decl_simple_lock_data(,vm_page_bucket_lock) #if MACH_PAGE_HASH_STATS /* This routine is only for debug. It is intended to be called by * hand by a developer using a kernel debugger. This routine prints * out vm_page_hash table statistics to the kernel debug console. */ void hash_debug(void) { int i; int numbuckets = 0; int highsum = 0; int maxdepth = 0; for (i = 0; i < vm_page_bucket_count; i++) { if (vm_page_buckets[i].hi_count) { numbuckets++; highsum += vm_page_buckets[i].hi_count; if (vm_page_buckets[i].hi_count > maxdepth) maxdepth = vm_page_buckets[i].hi_count; } } printf("Total number of buckets: %d\n", vm_page_bucket_count); printf("Number used buckets: %d = %d%%\n", numbuckets, 100*numbuckets/vm_page_bucket_count); printf("Number unused buckets: %d = %d%%\n", vm_page_bucket_count - numbuckets, 100*(vm_page_bucket_count-numbuckets)/vm_page_bucket_count); printf("Sum of bucket max depth: %d\n", highsum); printf("Average bucket depth: %d.%2d\n", highsum/vm_page_bucket_count, highsum%vm_page_bucket_count); printf("Maximum bucket depth: %d\n", maxdepth); } #endif /* MACH_PAGE_HASH_STATS */ /* * The virtual page size is currently implemented as a runtime * variable, but is constant once initialized using vm_set_page_size. * This initialization must be done in the machine-dependent * bootstrap sequence, before calling other machine-independent * initializations. * * All references to the virtual page size outside this * module must use the PAGE_SIZE, PAGE_MASK and PAGE_SHIFT * constants. */ #ifndef PAGE_SIZE_FIXED vm_size_t page_size = 4096; vm_size_t page_mask = 4095; int page_shift = 12; #endif /* PAGE_SIZE_FIXED */ /* * Resident page structures are initialized from * a template (see vm_page_alloc). * * When adding a new field to the virtual memory * object structure, be sure to add initialization * (see vm_page_bootstrap). */ struct vm_page vm_page_template; /* * Resident pages that represent real memory * are allocated from a free list. */ vm_page_t vm_page_queue_free; vm_page_t vm_page_queue_fictitious; decl_mutex_data(,vm_page_queue_free_lock) unsigned int vm_page_free_wanted; int vm_page_free_count; int vm_page_fictitious_count; unsigned int vm_page_free_count_minimum; /* debugging */ /* * Occasionally, the virtual memory system uses * resident page structures that do not refer to * real pages, for example to leave a page with * important state information in the VP table. * * These page structures are allocated the way * most other kernel structures are. */ zone_t vm_page_zone; decl_mutex_data(,vm_page_alloc_lock) /* * Fictitious pages don't have a physical address, * but we must initialize phys_addr to something. * For debugging, this should be a strange value * that the pmap module can recognize in assertions. */ vm_offset_t vm_page_fictitious_addr = (vm_offset_t) -1; /* * Resident page structures are also chained on * queues that are used by the page replacement * system (pageout daemon). These queues are * defined here, but are shared by the pageout * module. */ queue_head_t vm_page_queue_active; queue_head_t vm_page_queue_inactive; decl_mutex_data(,vm_page_queue_lock) int vm_page_active_count; int vm_page_inactive_count; int vm_page_wire_count; int vm_page_gobble_count = 0; int vm_page_wire_count_warning = 0; int vm_page_gobble_count_warning = 0; /* the following fields are protected by the vm_page_queue_lock */ queue_head_t vm_page_queue_limbo; int vm_page_limbo_count = 0; /* total pages in limbo */ int vm_page_limbo_real_count = 0; /* real pages in limbo */ int vm_page_pin_count = 0; /* number of pinned pages */ decl_simple_lock_data(,vm_page_preppin_lock) /* * Several page replacement parameters are also * shared with this module, so that page allocation * (done here in vm_page_alloc) can trigger the * pageout daemon. */ int vm_page_free_target = 0; int vm_page_free_min = 0; int vm_page_inactive_target = 0; int vm_page_free_reserved = 0; int vm_page_laundry_count = 0; /* * The VM system has a couple of heuristics for deciding * that pages are "uninteresting" and should be placed * on the inactive queue as likely candidates for replacement. * These variables let the heuristics be controlled at run-time * to make experimentation easier. */ boolean_t vm_page_deactivate_hint = TRUE; /* * vm_set_page_size: * * Sets the page size, perhaps based upon the memory * size. Must be called before any use of page-size * dependent functions. * * Sets page_shift and page_mask from page_size. */ void vm_set_page_size(void) { #ifndef PAGE_SIZE_FIXED page_mask = page_size - 1; if ((page_mask & page_size) != 0) panic("vm_set_page_size: page size not a power of two"); for (page_shift = 0; ; page_shift++) if ((1 << page_shift) == page_size) break; #endif /* PAGE_SIZE_FIXED */ } /* * vm_page_bootstrap: * * Initializes the resident memory module. * * Allocates memory for the page cells, and * for the object/offset-to-page hash table headers. * Each page cell is initialized and placed on the free list. * Returns the range of available kernel virtual memory. */ void vm_page_bootstrap( vm_offset_t *startp, vm_offset_t *endp) { register vm_page_t m; int i; unsigned int log1; unsigned int log2; unsigned int size; /* * Initialize the vm_page template. */ m = &vm_page_template; m->object = VM_OBJECT_NULL; /* reset later */ m->offset = 0; /* reset later */ m->wire_count = 0; m->inactive = FALSE; m->active = FALSE; m->laundry = FALSE; m->free = FALSE; m->reference = FALSE; m->pageout = FALSE; m->list_req_pending = FALSE; m->busy = TRUE; m->wanted = FALSE; m->tabled = FALSE; m->fictitious = FALSE; m->private = FALSE; m->absent = FALSE; m->error = FALSE; m->dirty = FALSE; m->cleaning = FALSE; m->precious = FALSE; m->clustered = FALSE; m->lock_supplied = FALSE; m->unusual = FALSE; m->restart = FALSE; m->limbo = FALSE; m->phys_addr = 0; /* reset later */ m->page_lock = VM_PROT_NONE; m->unlock_request = VM_PROT_NONE; m->page_error = KERN_SUCCESS; /* * Initialize the page queues. */ mutex_init(&vm_page_queue_free_lock, ETAP_VM_PAGEQ_FREE); mutex_init(&vm_page_queue_lock, ETAP_VM_PAGEQ); simple_lock_init(&vm_page_preppin_lock, ETAP_VM_PREPPIN); vm_page_queue_free = VM_PAGE_NULL; vm_page_queue_fictitious = VM_PAGE_NULL; queue_init(&vm_page_queue_active); queue_init(&vm_page_queue_inactive); queue_init(&vm_page_queue_limbo); vm_page_free_wanted = 0; /* * Steal memory for the map and zone subsystems. */ vm_map_steal_memory(); zone_steal_memory(); /* * Allocate (and initialize) the virtual-to-physical * table hash buckets. * * The number of buckets should be a power of two to * get a good hash function. The following computation * chooses the first power of two that is greater * than the number of physical pages in the system. */ simple_lock_init(&vm_page_bucket_lock, ETAP_VM_BUCKET); if (vm_page_bucket_count == 0) { unsigned int npages = pmap_free_pages(); vm_page_bucket_count = 1; while (vm_page_bucket_count < npages) vm_page_bucket_count <<= 1; } vm_page_hash_mask = vm_page_bucket_count - 1; /* * Calculate object shift value for hashing algorithm: * O = log2(sizeof(struct vm_object)) * B = log2(vm_page_bucket_count) * hash shifts the object left by * B/2 - O */ size = vm_page_bucket_count; for (log1 = 0; size > 1; log1++) size /= 2; size = sizeof(struct vm_object); for (log2 = 0; size > 1; log2++) size /= 2; vm_page_hash_shift = log1/2 - log2 + 1; if (vm_page_hash_mask & vm_page_bucket_count) printf("vm_page_bootstrap: WARNING -- strange page hash\n"); vm_page_buckets = (vm_page_bucket_t *) pmap_steal_memory(vm_page_bucket_count * sizeof(vm_page_bucket_t)); for (i = 0; i < vm_page_bucket_count; i++) { register vm_page_bucket_t *bucket = &vm_page_buckets[i]; bucket->pages = VM_PAGE_NULL; #if MACH_PAGE_HASH_STATS bucket->cur_count = 0; bucket->hi_count = 0; #endif /* MACH_PAGE_HASH_STATS */ } /* * Machine-dependent code allocates the resident page table. * It uses vm_page_init to initialize the page frames. * The code also returns to us the virtual space available * to the kernel. We don't trust the pmap module * to get the alignment right. */ pmap_startup(&virtual_space_start, &virtual_space_end); virtual_space_start = round_page(virtual_space_start); virtual_space_end = trunc_page(virtual_space_end); *startp = virtual_space_start; *endp = virtual_space_end; /* * Compute the initial "wire" count. * Up until now, the pages which have been set aside are not under * the VM system's control, so although they aren't explicitly * wired, they nonetheless can't be moved. At this moment, * all VM managed pages are "free", courtesy of pmap_startup. */ vm_page_wire_count = atop(mem_size) - vm_page_free_count; /* initial value */ printf("vm_page_bootstrap: %d free pages\n", vm_page_free_count); vm_page_free_count_minimum = vm_page_free_count; } #ifndef MACHINE_PAGES /* * We implement pmap_steal_memory and pmap_startup with the help * of two simpler functions, pmap_virtual_space and pmap_next_page. */ vm_offset_t pmap_steal_memory( vm_size_t size) { vm_offset_t addr, vaddr, paddr; /* * We round the size to a round multiple. */ size = (size + sizeof (void *) - 1) &~ (sizeof (void *) - 1); /* * If this is the first call to pmap_steal_memory, * we have to initialize ourself. */ if (virtual_space_start == virtual_space_end) { pmap_virtual_space(&virtual_space_start, &virtual_space_end); /* * The initial values must be aligned properly, and * we don't trust the pmap module to do it right. */ virtual_space_start = round_page(virtual_space_start); virtual_space_end = trunc_page(virtual_space_end); } /* * Allocate virtual memory for this request. */ addr = virtual_space_start; virtual_space_start += size; kprintf("pmap_steal_memory: %08X - %08X; size=%08X\n", addr, virtual_space_start, size); /* (TEST/DEBUG) */ /* * Allocate and map physical pages to back new virtual pages. */ for (vaddr = round_page(addr); vaddr < addr + size; vaddr += PAGE_SIZE) { if (!pmap_next_page(&paddr)) panic("pmap_steal_memory"); /* * XXX Logically, these mappings should be wired, * but some pmap modules barf if they are. */ pmap_enter(kernel_pmap, vaddr, paddr, VM_PROT_READ|VM_PROT_WRITE, FALSE); /* * Account for newly stolen memory */ vm_page_wire_count++; } return addr; } void pmap_startup( vm_offset_t *startp, vm_offset_t *endp) { unsigned int i, npages, pages_initialized; vm_page_t pages; vm_offset_t paddr; /* * We calculate how many page frames we will have * and then allocate the page structures in one chunk. */ npages = ((PAGE_SIZE * pmap_free_pages() + (round_page(virtual_space_start) - virtual_space_start)) / (PAGE_SIZE + sizeof *pages)); pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages); /* * Initialize the page frames. */ for (i = 0, pages_initialized = 0; i < npages; i++) { if (!pmap_next_page(&paddr)) break; vm_page_init(&pages[i], paddr); vm_page_pages++; pages_initialized++; } /* * Release pages in reverse order so that physical pages * initially get allocated in ascending addresses. This keeps * the devices (which must address physical memory) happy if * they require several consecutive pages. */ for (i = pages_initialized; i > 0; i--) { vm_page_release(&pages[i - 1]); } /* * We have to re-align virtual_space_start, * because pmap_steal_memory has been using it. */ virtual_space_start = round_page(virtual_space_start); *startp = virtual_space_start; *endp = virtual_space_end; } #endif /* MACHINE_PAGES */ /* * Routine: vm_page_module_init * Purpose: * Second initialization pass, to be done after * the basic VM system is ready. */ void vm_page_module_init(void) { vm_page_zone = zinit((vm_size_t) sizeof(struct vm_page), 0, PAGE_SIZE, "vm pages"); #if ZONE_DEBUG zone_debug_disable(vm_page_zone); #endif /* ZONE_DEBUG */ zone_change(vm_page_zone, Z_EXPAND, FALSE); zone_change(vm_page_zone, Z_EXHAUST, TRUE); zone_change(vm_page_zone, Z_FOREIGN, TRUE); /* * Adjust zone statistics to account for the real pages allocated * in vm_page_create(). [Q: is this really what we want?] */ vm_page_zone->count += vm_page_pages; vm_page_zone->cur_size += vm_page_pages * vm_page_zone->elem_size; mutex_init(&vm_page_alloc_lock, ETAP_VM_PAGE_ALLOC); } /* * Routine: vm_page_create * Purpose: * After the VM system is up, machine-dependent code * may stumble across more physical memory. For example, * memory that it was reserving for a frame buffer. * vm_page_create turns this memory into available pages. */ void vm_page_create( vm_offset_t start, vm_offset_t end) { vm_offset_t paddr; vm_page_t m; for (paddr = round_page(start); paddr < trunc_page(end); paddr += PAGE_SIZE) { while ((m = (vm_page_t) vm_page_grab_fictitious()) == VM_PAGE_NULL) vm_page_more_fictitious(); vm_page_init(m, paddr); vm_page_pages++; vm_page_release(m); } } /* * vm_page_hash: * * Distributes the object/offset key pair among hash buckets. * * NOTE: To get a good hash function, the bucket count should * be a power of two. */ #define vm_page_hash(object, offset) (\ ( ((natural_t)(vm_offset_t)object<tabled) panic("vm_page_insert"); assert(!object->internal || offset < object->size); /* only insert "pageout" pages into "pageout" objects, * and normal pages into normal objects */ assert(object->pageout == mem->pageout); /* * Record the object/offset pair in this page */ mem->object = object; mem->offset = offset; /* * Insert it into the object_object/offset hash table */ bucket = &vm_page_buckets[vm_page_hash(object, offset)]; simple_lock(&vm_page_bucket_lock); mem->next = bucket->pages; bucket->pages = mem; #if MACH_PAGE_HASH_STATS if (++bucket->cur_count > bucket->hi_count) bucket->hi_count = bucket->cur_count; #endif /* MACH_PAGE_HASH_STATS */ simple_unlock(&vm_page_bucket_lock); /* * Now link into the object's list of backed pages. */ queue_enter(&object->memq, mem, vm_page_t, listq); mem->tabled = TRUE; /* * Show that the object has one more resident page. */ object->resident_page_count++; } /* * vm_page_replace: * * Exactly like vm_page_insert, except that we first * remove any existing page at the given offset in object. * * The object and page queues must be locked. */ void vm_page_replace( register vm_page_t mem, register vm_object_t object, register vm_object_offset_t offset) { register vm_page_bucket_t *bucket; VM_PAGE_CHECK(mem); if (mem->tabled) panic("vm_page_replace"); /* * Record the object/offset pair in this page */ mem->object = object; mem->offset = offset; /* * Insert it into the object_object/offset hash table, * replacing any page that might have been there. */ bucket = &vm_page_buckets[vm_page_hash(object, offset)]; simple_lock(&vm_page_bucket_lock); if (bucket->pages) { vm_page_t *mp = &bucket->pages; register vm_page_t m = *mp; do { if (m->object == object && m->offset == offset) { /* * Remove page from bucket and from object, * and return it to the free list. */ *mp = m->next; queue_remove(&object->memq, m, vm_page_t, listq); m->tabled = FALSE; object->resident_page_count--; /* * Return page to the free list. * Note the page is not tabled now, so this * won't self-deadlock on the bucket lock. */ vm_page_free(m); break; } mp = &m->next; } while (m = *mp); mem->next = bucket->pages; } else { mem->next = VM_PAGE_NULL; } bucket->pages = mem; simple_unlock(&vm_page_bucket_lock); /* * Now link into the object's list of backed pages. */ queue_enter(&object->memq, mem, vm_page_t, listq); mem->tabled = TRUE; /* * And show that the object has one more resident * page. */ object->resident_page_count++; } /* * vm_page_remove: [ internal use only ] * * Removes the given mem entry from the object/offset-page * table and the object page list. * * The object and page must be locked. */ void vm_page_remove( register vm_page_t mem) { register vm_page_bucket_t *bucket; register vm_page_t this; XPR(XPR_VM_PAGE, "vm_page_remove, object 0x%X offset 0x%X page 0x%X\n", (integer_t)mem->object, (integer_t)mem->offset, (integer_t)mem, 0,0); assert(mem->tabled); assert(!mem->cleaning); VM_PAGE_CHECK(mem); /* * Remove from the object_object/offset hash table */ bucket = &vm_page_buckets[vm_page_hash(mem->object, mem->offset)]; simple_lock(&vm_page_bucket_lock); if ((this = bucket->pages) == mem) { /* optimize for common case */ bucket->pages = mem->next; } else { register vm_page_t *prev; for (prev = &this->next; (this = *prev) != mem; prev = &this->next) continue; *prev = this->next; } #if MACH_PAGE_HASH_STATS bucket->cur_count--; #endif /* MACH_PAGE_HASH_STATS */ simple_unlock(&vm_page_bucket_lock); /* * Now remove from the object's list of backed pages. */ queue_remove(&mem->object->memq, mem, vm_page_t, listq); /* * And show that the object has one fewer resident * page. */ mem->object->resident_page_count--; mem->tabled = FALSE; mem->object = VM_OBJECT_NULL; mem->offset = 0; } /* * vm_page_lookup: * * Returns the page associated with the object/offset * pair specified; if none is found, VM_PAGE_NULL is returned. * * The object must be locked. No side effects. */ vm_page_t vm_page_lookup( register vm_object_t object, register vm_object_offset_t offset) { register vm_page_t mem; register vm_page_bucket_t *bucket; /* * Search the hash table for this object/offset pair */ bucket = &vm_page_buckets[vm_page_hash(object, offset)]; simple_lock(&vm_page_bucket_lock); for (mem = bucket->pages; mem != VM_PAGE_NULL; mem = mem->next) { VM_PAGE_CHECK(mem); if ((mem->object == object) && (mem->offset == offset)) break; } simple_unlock(&vm_page_bucket_lock); return(mem); } /* * vm_page_rename: * * Move the given memory entry from its * current object to the specified target object/offset. * * The object must be locked. */ void vm_page_rename( register vm_page_t mem, register vm_object_t new_object, vm_object_offset_t new_offset) { assert(mem->object != new_object); /* * Changes to mem->object require the page lock because * the pageout daemon uses that lock to get the object. */ XPR(XPR_VM_PAGE, "vm_page_rename, new object 0x%X, offset 0x%X page 0x%X\n", (integer_t)new_object, (integer_t)new_offset, (integer_t)mem, 0,0); vm_page_lock_queues(); vm_page_remove(mem); vm_page_insert(mem, new_object, new_offset); vm_page_unlock_queues(); } /* * vm_page_init: * * Initialize the fields in a new page. * This takes a structure with random values and initializes it * so that it can be given to vm_page_release or vm_page_insert. */ void vm_page_init( vm_page_t mem, vm_offset_t phys_addr) { *mem = vm_page_template; mem->phys_addr = phys_addr; } /* * vm_page_grab_fictitious: * * Remove a fictitious page from the free list. * Returns VM_PAGE_NULL if there are no free pages. */ int c_vm_page_grab_fictitious = 0; int c_vm_page_release_fictitious = 0; int c_vm_page_more_fictitious = 0; vm_page_t vm_page_grab_fictitious(void) { register vm_page_t m; m = (vm_page_t)zget(vm_page_zone); if (m) { m->free = FALSE; #if MACH_ASSERT || ZONE_DEBUG vm_page_init(m, vm_page_fictitious_addr); m->fictitious = TRUE; #endif /* MACH_ASSERT || ZONE_DEBUG */ } c_vm_page_grab_fictitious++; return m; } /* * vm_page_release_fictitious: * * Release a fictitious page to the free list. */ void vm_page_release_fictitious( register vm_page_t m) { assert(!m->free); assert(m->busy); assert(m->fictitious); assert(m->phys_addr == vm_page_fictitious_addr); c_vm_page_release_fictitious++; if (m->free) panic("vm_page_release_fictitious"); m->free = TRUE; zfree(vm_page_zone, (vm_offset_t)m); } /* * vm_page_more_fictitious: * * Add more fictitious pages to the free list. * Allowed to block. This routine is way intimate * with the zones code, for several reasons: * 1. we need to carve some page structures out of physical * memory before zones work, so they _cannot_ come from * the zone_map. * 2. the zone needs to be collectable in order to prevent * growth without bound. These structures are used by * the device pager (by the hundreds and thousands), as * private pages for pageout, and as blocking pages for * pagein. Temporary bursts in demand should not result in * permanent allocation of a resource. * 3. To smooth allocation humps, we allocate single pages * with kernel_memory_allocate(), and cram them into the * zone. This also allows us to initialize the vm_page_t's * on the way into the zone, so that zget() always returns * an initialized structure. The zone free element pointer * and the free page pointer are both the first item in the * vm_page_t. * 4. By having the pages in the zone pre-initialized, we need * not keep 2 levels of lists. The garbage collector simply * scans our list, and reduces physical memory usage as it * sees fit. */ void vm_page_more_fictitious(void) { extern vm_map_t zone_map; register vm_page_t m; vm_offset_t addr; kern_return_t retval; int i; c_vm_page_more_fictitious++; /* this may free up some fictitious pages */ cleanup_limbo_queue(); /* * Allocate a single page from the zone_map. Do not wait if no physical * pages are immediately available, and do not zero the space. We need * our own blocking lock here to prevent having multiple, * simultaneous requests from piling up on the zone_map lock. Exactly * one (of our) threads should be potentially waiting on the map lock. * If winner is not vm-privileged, then the page allocation will fail, * and it will temporarily block here in the vm_page_wait(). */ mutex_lock(&vm_page_alloc_lock); /* * If another thread allocated space, just bail out now. */ if (zone_free_count(vm_page_zone) > 5) { /* * The number "5" is a small number that is larger than the * number of fictitious pages that any single caller will * attempt to allocate. Otherwise, a thread will attempt to * acquire a fictitious page (vm_page_grab_fictitious), fail, * release all of the resources and locks already acquired, * and then call this routine. This routine finds the pages * that the caller released, so fails to allocate new space. * The process repeats infinitely. The largest known number * of fictitious pages required in this manner is 2. 5 is * simply a somewhat larger number. */ mutex_unlock(&vm_page_alloc_lock); return; } if ((retval = kernel_memory_allocate(zone_map, &addr, PAGE_SIZE, VM_PROT_ALL, KMA_KOBJECT|KMA_NOPAGEWAIT)) != KERN_SUCCESS) { /* * No page was available. Tell the pageout daemon, drop the * lock to give another thread a chance at it, and * wait for the pageout daemon to make progress. */ mutex_unlock(&vm_page_alloc_lock); vm_page_wait(THREAD_UNINT); return; } /* * Initialize as many vm_page_t's as will fit on this page. This * depends on the zone code disturbing ONLY the first item of * each zone element. */ m = (vm_page_t)addr; for (i = PAGE_SIZE/sizeof(struct vm_page); i > 0; i--) { vm_page_init(m, vm_page_fictitious_addr); m->fictitious = TRUE; m++; } zcram(vm_page_zone, addr, PAGE_SIZE); mutex_unlock(&vm_page_alloc_lock); } /* * vm_page_convert: * * Attempt to convert a fictitious page into a real page. */ boolean_t vm_page_convert( register vm_page_t m) { register vm_page_t real_m; assert(m->busy); assert(m->fictitious); assert(!m->dirty); real_m = vm_page_grab(); if (real_m == VM_PAGE_NULL) return FALSE; m->phys_addr = real_m->phys_addr; m->fictitious = FALSE; vm_page_lock_queues(); if (m->active) vm_page_active_count++; else if (m->inactive) vm_page_inactive_count++; vm_page_unlock_queues(); real_m->phys_addr = vm_page_fictitious_addr; real_m->fictitious = TRUE; vm_page_release_fictitious(real_m); return TRUE; } /* * vm_pool_low(): * * Return true if it is not likely that a non-vm_privileged thread * can get memory without blocking. Advisory only, since the * situation may change under us. */ int vm_pool_low(void) { /* No locking, at worst we will fib. */ return( vm_page_free_count < vm_page_free_reserved ); } /* * vm_page_grab: * * Remove a page from the free list. * Returns VM_PAGE_NULL if the free list is too small. */ unsigned long vm_page_grab_count = 0; /* measure demand */ vm_page_t vm_page_grab(void) { register vm_page_t mem; mutex_lock(&vm_page_queue_free_lock); vm_page_grab_count++; /* * Optionally produce warnings if the wire or gobble * counts exceed some threshold. */ if (vm_page_wire_count_warning > 0 && vm_page_wire_count >= vm_page_wire_count_warning) { printf("mk: vm_page_grab(): high wired page count of %d\n", vm_page_wire_count); assert(vm_page_wire_count < vm_page_wire_count_warning); } if (vm_page_gobble_count_warning > 0 && vm_page_gobble_count >= vm_page_gobble_count_warning) { printf("mk: vm_page_grab(): high gobbled page count of %d\n", vm_page_gobble_count); assert(vm_page_gobble_count < vm_page_gobble_count_warning); } /* * Only let privileged threads (involved in pageout) * dip into the reserved pool. */ if ((vm_page_free_count < vm_page_free_reserved) && !current_thread()->vm_privilege) { mutex_unlock(&vm_page_queue_free_lock); mem = VM_PAGE_NULL; goto wakeup_pageout; } while (vm_page_queue_free == VM_PAGE_NULL) { printf("vm_page_grab: no free pages, trouble expected...\n"); mutex_unlock(&vm_page_queue_free_lock); VM_PAGE_WAIT(); mutex_lock(&vm_page_queue_free_lock); } if (--vm_page_free_count < vm_page_free_count_minimum) vm_page_free_count_minimum = vm_page_free_count; mem = vm_page_queue_free; vm_page_queue_free = (vm_page_t) mem->pageq.next; mem->free = FALSE; mutex_unlock(&vm_page_queue_free_lock); /* * Decide if we should poke the pageout daemon. * We do this if the free count is less than the low * water mark, or if the free count is less than the high * water mark (but above the low water mark) and the inactive * count is less than its target. * * We don't have the counts locked ... if they change a little, * it doesn't really matter. */ wakeup_pageout: if ((vm_page_free_count < vm_page_free_min) || ((vm_page_free_count < vm_page_free_target) && (vm_page_inactive_count < vm_page_inactive_target))) thread_wakeup((event_t) &vm_page_free_wanted); // dbgLog(mem->phys_addr, vm_page_free_count, vm_page_wire_count, 4); /* (TEST/DEBUG) */ return mem; } /* * vm_page_release: * * Return a page to the free list. */ void vm_page_release( register vm_page_t mem) { assert(!mem->private && !mem->fictitious); // dbgLog(mem->phys_addr, vm_page_free_count, vm_page_wire_count, 5); /* (TEST/DEBUG) */ mutex_lock(&vm_page_queue_free_lock); if (mem->free) panic("vm_page_release"); mem->free = TRUE; mem->pageq.next = (queue_entry_t) vm_page_queue_free; vm_page_queue_free = mem; vm_page_free_count++; /* * Check if we should wake up someone waiting for page. * But don't bother waking them unless they can allocate. * * We wakeup only one thread, to prevent starvation. * Because the scheduling system handles wait queues FIFO, * if we wakeup all waiting threads, one greedy thread * can starve multiple niceguy threads. When the threads * all wakeup, the greedy threads runs first, grabs the page, * and waits for another page. It will be the first to run * when the next page is freed. * * However, there is a slight danger here. * The thread we wake might not use the free page. * Then the other threads could wait indefinitely * while the page goes unused. To forestall this, * the pageout daemon will keep making free pages * as long as vm_page_free_wanted is non-zero. */ if ((vm_page_free_wanted > 0) && (vm_page_free_count >= vm_page_free_reserved)) { vm_page_free_wanted--; thread_wakeup_one((event_t) &vm_page_free_count); } mutex_unlock(&vm_page_queue_free_lock); } /* * Release a page to the limbo list. * Put real pages at the head of the queue, fictitious at the tail. * Page queues must be locked. */ void vm_page_release_limbo( register vm_page_t m) { assert(m->limbo); vm_page_limbo_count++; if (m->fictitious) { queue_enter(&vm_page_queue_limbo, m, vm_page_t, pageq); } else { vm_page_limbo_real_count++; queue_enter_first(&vm_page_queue_limbo, m, vm_page_t, pageq); } } /* * Exchange a real page in limbo (limbo_m) with a fictitious page (new_m). * The end result is that limbo_m is fictitious and still in limbo, and new_m * is the real page. The prep and pin counts remain with the page in limbo * although they will be briefly cleared by vm_page_init. This is OK since * there will be no interrupt-level interactions (the page is in limbo) and * vm_page_unprep must lock the page queues before changing the prep count. * * Page queues must be locked, and limbo_m must have been removed from its * object. */ void vm_page_limbo_exchange( register vm_page_t limbo_m, register vm_page_t new_m) { assert(limbo_m->limbo && !limbo_m->fictitious); assert(!limbo_m->tabled); assert(new_m->fictitious); *new_m = *limbo_m; vm_page_init(limbo_m, vm_page_fictitious_addr); limbo_m->fictitious = TRUE; limbo_m->limbo = TRUE; new_m->limbo = FALSE; limbo_m->prep_pin_count = new_m->prep_pin_count; new_m->prep_pin_count = 0; } /* * vm_page_wait: * * Wait for a page to become available. * If there are plenty of free pages, then we don't sleep. * * Returns: * TRUE: There may be another page, try again * FALSE: We were interrupted out of our wait, don't try again */ boolean_t vm_page_wait( int interruptible ) { /* * We can't use vm_page_free_reserved to make this * determination. Consider: some thread might * need to allocate two pages. The first allocation * succeeds, the second fails. After the first page is freed, * a call to vm_page_wait must really block. */ kern_return_t wait_result; mutex_lock(&vm_page_queue_free_lock); if (vm_page_free_count < vm_page_free_target) { if (vm_page_free_wanted++ == 0) thread_wakeup((event_t)&vm_page_free_wanted); assert_wait((event_t)&vm_page_free_count, interruptible); mutex_unlock(&vm_page_queue_free_lock); counter(c_vm_page_wait_block++); wait_result = thread_block((void (*)(void))0); return(wait_result == THREAD_AWAKENED); } else { mutex_unlock(&vm_page_queue_free_lock); return TRUE; } } /* * vm_page_alloc: * * Allocate and return a memory cell associated * with this VM object/offset pair. * * Object must be locked. */ vm_page_t vm_page_alloc( vm_object_t object, vm_object_offset_t offset) { register vm_page_t mem; mem = vm_page_grab(); if (mem == VM_PAGE_NULL) return VM_PAGE_NULL; vm_page_insert(mem, object, offset); return(mem); } int c_limbo_page_free = 0; /* debugging */ int c_limbo_convert = 0; /* debugging */ counter(unsigned int c_laundry_pages_freed = 0;) int vm_pagein_cluster_unused = 0; boolean_t vm_page_free_verify = FALSE; /* * vm_page_free: * * Returns the given page to the free list, * disassociating it with any VM object. * * Object and page queues must be locked prior to entry. */ void vm_page_free( register vm_page_t mem) { vm_object_t object = mem->object; assert(!mem->free); assert(!mem->cleaning); assert(!mem->pageout); assert(!vm_page_free_verify || pmap_verify_free(mem->phys_addr)); if (mem->tabled) vm_page_remove(mem); /* clears tabled, object, offset */ VM_PAGE_QUEUES_REMOVE(mem); /* clears active or inactive */ if (mem->clustered) { mem->clustered = FALSE; vm_pagein_cluster_unused++; } if (mem->wire_count) { if (!mem->private && !mem->fictitious) vm_page_wire_count--; mem->wire_count = 0; assert(!mem->gobbled); } else if (mem->gobbled) { if (!mem->private && !mem->fictitious) vm_page_wire_count--; vm_page_gobble_count--; } mem->gobbled = FALSE; if (mem->laundry) { extern int vm_page_laundry_min; vm_page_laundry_count--; mem->laundry = FALSE; /* laundry is now clear */ counter(++c_laundry_pages_freed); if (vm_page_laundry_count < vm_page_laundry_min) { vm_page_laundry_min = 0; thread_wakeup((event_t) &vm_page_laundry_count); } } mem->discard_request = FALSE; PAGE_WAKEUP(mem); /* clears wanted */ if (mem->absent) vm_object_absent_release(object); if (mem->limbo) { /* * The pageout daemon put this page into limbo and then freed * it. The page has already been removed from the object and * queues, so any attempt to look it up will fail. Put it * on the limbo queue; the pageout daemon will convert it to a * fictitious page and/or free the real one later. */ /* assert that it came from pageout daemon (how?) */ assert(!mem->fictitious && !mem->absent); c_limbo_page_free++; vm_page_release_limbo(mem); return; } assert(mem->prep_pin_count == 0); /* Some of these may be unnecessary */ mem->page_lock = 0; mem->unlock_request = 0; mem->busy = TRUE; mem->absent = FALSE; mem->error = FALSE; mem->dirty = FALSE; mem->precious = FALSE; mem->reference = FALSE; mem->page_error = KERN_SUCCESS; if (mem->private) { mem->private = FALSE; mem->fictitious = TRUE; mem->phys_addr = vm_page_fictitious_addr; } if (mem->fictitious) { vm_page_release_fictitious(mem); } else { vm_page_init(mem, mem->phys_addr); vm_page_release(mem); } } /* * vm_page_wire: * * Mark this page as wired down by yet * another map, removing it from paging queues * as necessary. * * The page's object and the page queues must be locked. */ void vm_page_wire( register vm_page_t mem) { // dbgLog(current_act(), mem->offset, mem->object, 1); /* (TEST/DEBUG) */ VM_PAGE_CHECK(mem); if (mem->wire_count == 0) { VM_PAGE_QUEUES_REMOVE(mem); if (!mem->private && !mem->fictitious && !mem->gobbled) vm_page_wire_count++; if (mem->gobbled) vm_page_gobble_count--; mem->gobbled = FALSE; } assert(!mem->gobbled); mem->wire_count++; } /* * vm_page_gobble: * * Mark this page as consumed by the vm/ipc/xmm subsystems. * * Called only for freshly vm_page_grab()ed pages - w/ nothing locked. */ void vm_page_gobble( register vm_page_t mem) { vm_page_lock_queues(); VM_PAGE_CHECK(mem); assert(!mem->gobbled); assert(mem->wire_count == 0); if (!mem->gobbled && mem->wire_count == 0) { if (!mem->private && !mem->fictitious) vm_page_wire_count++; } vm_page_gobble_count++; mem->gobbled = TRUE; vm_page_unlock_queues(); } /* * vm_page_unwire: * * Release one wiring of this page, potentially * enabling it to be paged again. * * The page's object and the page queues must be locked. */ void vm_page_unwire( register vm_page_t mem) { // dbgLog(current_act(), mem->offset, mem->object, 0); /* (TEST/DEBUG) */ VM_PAGE_CHECK(mem); assert(mem->wire_count > 0); if (--mem->wire_count == 0) { assert(!mem->private && !mem->fictitious); vm_page_wire_count--; queue_enter(&vm_page_queue_active, mem, vm_page_t, pageq); vm_page_active_count++; mem->active = TRUE; mem->reference = TRUE; } } /* * vm_page_deactivate: * * Returns the given page to the inactive list, * indicating that no physical maps have access * to this page. [Used by the physical mapping system.] * * The page queues must be locked. */ void vm_page_deactivate( register vm_page_t m) { VM_PAGE_CHECK(m); // dbgLog(m->phys_addr, vm_page_free_count, vm_page_wire_count, 6); /* (TEST/DEBUG) */ /* * This page is no longer very interesting. If it was * interesting (active or inactive/referenced), then we * clear the reference bit and (re)enter it in the * inactive queue. Note wired pages should not have * their reference bit cleared. */ if (m->gobbled) { /* can this happen? */ assert(m->wire_count == 0); if (!m->private && !m->fictitious) vm_page_wire_count--; vm_page_gobble_count--; m->gobbled = FALSE; } if (m->private || (m->wire_count != 0)) return; if (m->active || (m->inactive && m->reference)) { if (!m->fictitious && !m->absent) pmap_clear_reference(m->phys_addr); m->reference = FALSE; VM_PAGE_QUEUES_REMOVE(m); } if (m->wire_count == 0 && !m->inactive) { queue_enter(&vm_page_queue_inactive, m, vm_page_t, pageq); m->inactive = TRUE; if (!m->fictitious) vm_page_inactive_count++; } } /* * vm_page_activate: * * Put the specified page on the active list (if appropriate). * * The page queues must be locked. */ void vm_page_activate( register vm_page_t m) { VM_PAGE_CHECK(m); if (m->gobbled) { assert(m->wire_count == 0); if (!m->private && !m->fictitious) vm_page_wire_count--; vm_page_gobble_count--; m->gobbled = FALSE; } if (m->private) return; if (m->inactive) { queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq); if (!m->fictitious) vm_page_inactive_count--; m->inactive = FALSE; } if (m->wire_count == 0) { if (m->active) panic("vm_page_activate: already active"); queue_enter(&vm_page_queue_active, m, vm_page_t, pageq); m->active = TRUE; m->reference = TRUE; if (!m->fictitious) vm_page_active_count++; } } /* * vm_page_part_zero_fill: * * Zero-fill a part of the page. */ void vm_page_part_zero_fill( vm_page_t m, vm_offset_t m_pa, vm_size_t len) { vm_page_t tmp; VM_PAGE_CHECK(m); #ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED pmap_zero_part_page(m->phys_addr, m_pa, len); #else while (1) { tmp = vm_page_grab(); if (tmp == VM_PAGE_NULL) { vm_page_wait(THREAD_UNINT); continue; } break; } vm_page_zero_fill(tmp); if(m_pa != 0) { vm_page_part_copy(m, 0, tmp, 0, m_pa); } if((m_pa + len) < PAGE_SIZE) { vm_page_part_copy(m, m_pa + len, tmp, m_pa + len, PAGE_SIZE - (m_pa + len)); } vm_page_copy(tmp,m); vm_page_lock_queues(); vm_page_free(tmp); vm_page_unlock_queues(); #endif } /* * vm_page_zero_fill: * * Zero-fill the specified page. */ void vm_page_zero_fill( vm_page_t m) { XPR(XPR_VM_PAGE, "vm_page_zero_fill, object 0x%X offset 0x%X page 0x%X\n", (integer_t)m->object, (integer_t)m->offset, (integer_t)m, 0,0); VM_PAGE_CHECK(m); pmap_zero_page(m->phys_addr); } /* * vm_page_part_copy: * * copy part of one page to another */ void vm_page_part_copy( vm_page_t src_m, vm_offset_t src_pa, vm_page_t dst_m, vm_offset_t dst_pa, vm_size_t len) { VM_PAGE_CHECK(src_m); VM_PAGE_CHECK(dst_m); pmap_copy_part_page(src_m->phys_addr, src_pa, dst_m->phys_addr, dst_pa, len); } /* * vm_page_copy: * * Copy one page to another */ void vm_page_copy( vm_page_t src_m, vm_page_t dest_m) { XPR(XPR_VM_PAGE, "vm_page_copy, object 0x%X offset 0x%X to object 0x%X offset 0x%X\n", (integer_t)src_m->object, src_m->offset, (integer_t)dest_m->object, dest_m->offset, 0); VM_PAGE_CHECK(src_m); VM_PAGE_CHECK(dest_m); pmap_copy_page(src_m->phys_addr, dest_m->phys_addr); } /* * Limbo pages are placed on the limbo queue to await their prep count * going to zero. A page is put into limbo by the pageout daemon. If the * page is real, then the pageout daemon did not need to page out the page, * it just freed it. When the prep_pin_count is zero the page can be freed. * Real pages with a non-zero prep count are converted to fictitious pages * so that the memory can be reclaimed; the fictitious page will remain on * the limbo queue until its prep count reaches zero. * * cleanup_limbo_queue is called by vm_page_more_fictitious and the pageout * daemon since it can free both real and fictitious pages. * It returns the number of fictitious pages freed. */ void cleanup_limbo_queue(void) { register vm_page_t free_m, m; vm_offset_t phys_addr; vm_page_lock_queues(); assert(vm_page_limbo_count >= vm_page_limbo_real_count); /* * first free up all pages with prep/pin counts of zero. This * may free both real and fictitious pages, which may be needed * later to convert real ones. */ m = (vm_page_t)queue_first(&vm_page_queue_limbo); while (!queue_end(&vm_page_queue_limbo, (queue_entry_t)m)) { if (m->prep_pin_count == 0) { free_m = m; m = (vm_page_t)queue_next(&m->pageq); queue_remove(&vm_page_queue_limbo, free_m, vm_page_t, pageq); vm_page_limbo_count--; if (!free_m->fictitious) vm_page_limbo_real_count--; free_m->limbo = FALSE; vm_page_free(free_m); assert(vm_page_limbo_count >= 0); assert(vm_page_limbo_real_count >= 0); } else { m = (vm_page_t)queue_next(&m->pageq); } } /* * now convert any remaining real pages to fictitious and free the * real ones. */ while (vm_page_limbo_real_count > 0) { queue_remove_first(&vm_page_queue_limbo, m, vm_page_t, pageq); assert(!m->fictitious); assert(m->limbo); /* * Try to get a fictitious page. If impossible, * requeue the real one and give up. */ free_m = vm_page_grab_fictitious(); if (free_m == VM_PAGE_NULL) { queue_enter_first(&vm_page_queue_limbo, m, vm_page_t, pageq); break; } c_limbo_convert++; vm_page_limbo_exchange(m, free_m); assert(m->limbo && m->fictitious); assert(!free_m->limbo && !free_m->fictitious); queue_enter(&vm_page_queue_limbo, m, vm_page_t, pageq); vm_page_free(free_m); vm_page_limbo_real_count--; } vm_page_unlock_queues(); } /* * Increment prep_count on a page. * Must be called in thread context. Page must not disappear: object * must be locked. */ kern_return_t vm_page_prep( register vm_page_t m) { kern_return_t retval = KERN_SUCCESS; assert(m != VM_PAGE_NULL); vm_page_lock_queues(); if (!m->busy && !m->error && !m->fictitious && !m->absent) { if (m->prep_pin_count != 0) { vm_page_pin_lock(); m->prep_count++; vm_page_pin_unlock(); } else { m->prep_count++; } assert(m->prep_count != 0); /* check for wraparound */ } else { retval = KERN_FAILURE; } vm_page_unlock_queues(); return retval; } /* * Pin a page (increment pin count). * Must have been previously prepped. * * MUST BE CALLED AT SPLVM. * * May be called from thread or interrupt context. * If page is in "limbo" it cannot be pinned. */ kern_return_t vm_page_pin( register vm_page_t m) { kern_return_t retval = KERN_SUCCESS; assert(m != VM_PAGE_NULL); vm_page_pin_lock(); if (m->limbo || m->prep_count == 0) { retval = KERN_FAILURE; } else { assert(!m->fictitious); if (m->pin_count == 0) vm_page_pin_count++; m->pin_count++; } vm_page_pin_unlock(); return retval; } /* * Unprep a page (decrement prep count). * Must have been previously prepped. * Called to decrement prep count after an attempt to pin failed. * Must be called from thread context. */ kern_return_t vm_page_unprep( register vm_page_t m) { kern_return_t retval = KERN_SUCCESS; assert(m != VM_PAGE_NULL); vm_page_lock_queues(); vm_page_pin_lock(); assert(m->prep_count != 0); if (m->prep_count == 0) retval = KERN_FAILURE; /* shouldn't happen */ else m->prep_count--; vm_page_pin_unlock(); vm_page_unlock_queues(); return retval; } /* * Unpin a page: decrement pin AND prep counts. * Must have been previously prepped AND pinned. * * MUST BE CALLED AT SPLVM. * * May be called from thread or interrupt context. */ kern_return_t vm_page_unpin( register vm_page_t m) { kern_return_t retval = KERN_SUCCESS; assert(m != VM_PAGE_NULL); vm_page_pin_lock(); assert(m->prep_count != 0 && m->pin_count != 0); assert(m->prep_count >= m->pin_count); assert(!m->limbo && !m->fictitious); if (m->prep_count != 0 && m->pin_count != 0) { m->prep_count--; m->pin_count--; if (m->pin_count == 0) vm_page_pin_count--; } else { retval = KERN_FAILURE; /* shouldn't happen */ } vm_page_pin_unlock(); return retval; } /* * Currently, this is a primitive allocator that grabs * free pages from the system, sorts them by physical * address, then searches for a region large enough to * satisfy the user's request. * * Additional levels of effort: * + steal clean active/inactive pages * + force pageouts of dirty pages * + maintain a map of available physical * memory */ #define SET_NEXT_PAGE(m,n) ((m)->pageq.next = (struct queue_entry *) (n)) #if MACH_ASSERT int vm_page_verify_contiguous( vm_page_t pages, unsigned int npages); #endif /* MACH_ASSERT */ cpm_counter(unsigned int vpfls_pages_handled = 0;) cpm_counter(unsigned int vpfls_head_insertions = 0;) cpm_counter(unsigned int vpfls_tail_insertions = 0;) cpm_counter(unsigned int vpfls_general_insertions = 0;) cpm_counter(unsigned int vpfc_failed = 0;) cpm_counter(unsigned int vpfc_satisfied = 0;) /* * Sort free list by ascending physical address, * using a not-particularly-bright sort algorithm. * Caller holds vm_page_queue_free_lock. */ static void vm_page_free_list_sort(void) { vm_page_t sort_list; vm_page_t sort_list_end; vm_page_t m, m1, *prev, next_m; vm_offset_t addr; #if MACH_ASSERT unsigned int npages; int old_free_count; #endif /* MACH_ASSERT */ #if MACH_ASSERT /* * Verify pages in the free list.. */ npages = 0; for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) ++npages; if (npages != vm_page_free_count) panic("vm_sort_free_list: prelim: npages %d free_count %d", npages, vm_page_free_count); old_free_count = vm_page_free_count; #endif /* MACH_ASSERT */ sort_list = sort_list_end = vm_page_queue_free; m = NEXT_PAGE(vm_page_queue_free); SET_NEXT_PAGE(vm_page_queue_free, VM_PAGE_NULL); cpm_counter(vpfls_pages_handled = 0); while (m != VM_PAGE_NULL) { cpm_counter(++vpfls_pages_handled); next_m = NEXT_PAGE(m); if (m->phys_addr < sort_list->phys_addr) { cpm_counter(++vpfls_head_insertions); SET_NEXT_PAGE(m, sort_list); sort_list = m; } else if (m->phys_addr > sort_list_end->phys_addr) { cpm_counter(++vpfls_tail_insertions); SET_NEXT_PAGE(sort_list_end, m); SET_NEXT_PAGE(m, VM_PAGE_NULL); sort_list_end = m; } else { cpm_counter(++vpfls_general_insertions); /* general sorted list insertion */ prev = &sort_list; for (m1=sort_list; m1!=VM_PAGE_NULL; m1=NEXT_PAGE(m1)) { if (m1->phys_addr > m->phys_addr) { if (*prev != m1) panic("vm_sort_free_list: ugh"); SET_NEXT_PAGE(m, *prev); *prev = m; break; } prev = (vm_page_t *) &m1->pageq.next; } } m = next_m; } #if MACH_ASSERT /* * Verify that pages are sorted into ascending order. */ for (m = sort_list, npages = 0; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) { if (m != sort_list && m->phys_addr <= addr) { printf("m 0x%x addr 0x%x\n", m, addr); panic("vm_sort_free_list"); } addr = m->phys_addr; ++npages; } if (old_free_count != vm_page_free_count) panic("vm_sort_free_list: old_free %d free_count %d", old_free_count, vm_page_free_count); if (npages != vm_page_free_count) panic("vm_sort_free_list: npages %d free_count %d", npages, vm_page_free_count); #endif /* MACH_ASSERT */ vm_page_queue_free = sort_list; } #if MACH_ASSERT /* * Check that the list of pages is ordered by * ascending physical address and has no holes. */ int vm_page_verify_contiguous( vm_page_t pages, unsigned int npages) { register vm_page_t m; unsigned int page_count; vm_offset_t prev_addr; prev_addr = pages->phys_addr; page_count = 1; for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) { if (m->phys_addr != prev_addr + page_size) { printf("m 0x%x prev_addr 0x%x, current addr 0x%x\n", m, prev_addr, m->phys_addr); printf("pages 0x%x page_count %d\n", pages, page_count); panic("vm_page_verify_contiguous: not contiguous!"); } prev_addr = m->phys_addr; ++page_count; } if (page_count != npages) { printf("pages 0x%x actual count 0x%x but requested 0x%x\n", pages, page_count, npages); panic("vm_page_verify_contiguous: count error"); } return 1; } #endif /* MACH_ASSERT */ /* * Find a region large enough to contain at least npages * of contiguous physical memory. * * Requirements: * - Called while holding vm_page_queue_free_lock. * - Doesn't respect vm_page_free_reserved; caller * must not ask for more pages than are legal to grab. * * Returns a pointer to a list of gobbled pages or VM_PAGE_NULL. * */ static vm_page_t vm_page_find_contiguous( int npages) { vm_page_t m, *contig_prev, *prev_ptr; vm_offset_t prev_addr; unsigned int contig_npages; vm_page_t list; if (npages < 1) return VM_PAGE_NULL; prev_addr = vm_page_queue_free->phys_addr - (page_size + 1); prev_ptr = &vm_page_queue_free; for (m = vm_page_queue_free; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) { if (m->phys_addr != prev_addr + page_size) { /* * Whoops! Pages aren't contiguous. Start over. */ contig_npages = 0; contig_prev = prev_ptr; } if (++contig_npages == npages) { /* * Chop these pages out of the free list. * Mark them all as gobbled. */ list = *contig_prev; *contig_prev = NEXT_PAGE(m); SET_NEXT_PAGE(m, VM_PAGE_NULL); for (m = list; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) { assert(m->free); assert(!m->wanted); m->free = FALSE; m->gobbled = TRUE; } vm_page_free_count -= npages; if (vm_page_free_count < vm_page_free_count_minimum) vm_page_free_count_minimum = vm_page_free_count; vm_page_wire_count += npages; vm_page_gobble_count += npages; cpm_counter(++vpfc_satisfied); assert(vm_page_verify_contiguous(list, contig_npages)); return list; } assert(contig_npages < npages); prev_ptr = (vm_page_t *) &m->pageq.next; prev_addr = m->phys_addr; } cpm_counter(++vpfc_failed); return VM_PAGE_NULL; } /* * Allocate a list of contiguous, wired pages. */ kern_return_t cpm_allocate( vm_size_t size, vm_page_t *list, boolean_t wire) { register vm_page_t m; vm_page_t *first_contig; vm_page_t free_list, pages; unsigned int npages, n1pages; int vm_pages_available; if (size % page_size != 0) return KERN_INVALID_ARGUMENT; vm_page_lock_queues(); mutex_lock(&vm_page_queue_free_lock); /* * Should also take active and inactive pages * into account... One day... */ vm_pages_available = vm_page_free_count - vm_page_free_reserved; if (size > vm_pages_available * page_size) { mutex_unlock(&vm_page_queue_free_lock); return KERN_RESOURCE_SHORTAGE; } vm_page_free_list_sort(); npages = size / page_size; /* * Obtain a pointer to a subset of the free * list large enough to satisfy the request; * the region will be physically contiguous. */ pages = vm_page_find_contiguous(npages); if (pages == VM_PAGE_NULL) { mutex_unlock(&vm_page_queue_free_lock); vm_page_unlock_queues(); return KERN_NO_SPACE; } mutex_unlock(&vm_page_queue_free_lock); /* * Walk the returned list, wiring the pages. */ if (wire == TRUE) for (m = pages; m != VM_PAGE_NULL; m = NEXT_PAGE(m)) { /* * Essentially inlined vm_page_wire. */ assert(!m->active); assert(!m->inactive); assert(!m->private); assert(!m->fictitious); assert(m->wire_count == 0); assert(m->gobbled); m->gobbled = FALSE; m->wire_count++; --vm_page_gobble_count; } vm_page_unlock_queues(); /* * The CPM pages should now be available and * ordered by ascending physical address. */ assert(vm_page_verify_contiguous(pages, npages)); *list = pages; return KERN_SUCCESS; } #include #if MACH_VM_DEBUG #include #include /* * Routine: vm_page_info * Purpose: * Return information about the global VP table. * Fills the buffer with as much information as possible * and returns the desired size of the buffer. * Conditions: * Nothing locked. The caller should provide * possibly-pageable memory. */ unsigned int vm_page_info( hash_info_bucket_t *info, unsigned int count) { int i; if (vm_page_bucket_count < count) count = vm_page_bucket_count; for (i = 0; i < count; i++) { vm_page_bucket_t *bucket = &vm_page_buckets[i]; unsigned int bucket_count = 0; vm_page_t m; simple_lock(&vm_page_bucket_lock); for (m = bucket->pages; m != VM_PAGE_NULL; m = m->next) bucket_count++; simple_unlock(&vm_page_bucket_lock); /* don't touch pageable memory while holding locks */ info[i].hib_count = bucket_count; } return vm_page_bucket_count; } #endif /* MACH_VM_DEBUG */ #include #if MACH_KDB #include #include #define printf kdbprintf /* * Routine: vm_page_print [exported] */ void vm_page_print( vm_page_t p) { extern db_indent; iprintf("page 0x%x\n", p); db_indent += 2; iprintf("object=0x%x", p->object); printf(", offset=0x%x", p->offset); printf(", wire_count=%d", p->wire_count); printf(", prep_count=%d", p->prep_count); printf(", pin_count=%d\n", p->pin_count); iprintf("%sinactive, %sactive, %sgobbled, %slaundry, %sfree, %sref, %sdiscard\n", (p->inactive ? "" : "!"), (p->active ? "" : "!"), (p->gobbled ? "" : "!"), (p->laundry ? "" : "!"), (p->free ? "" : "!"), (p->reference ? "" : "!"), (p->discard_request ? "" : "!")); iprintf("%sbusy, %swanted, %stabled, %sfictitious, %sprivate, %sprecious\n", (p->busy ? "" : "!"), (p->wanted ? "" : "!"), (p->tabled ? "" : "!"), (p->fictitious ? "" : "!"), (p->private ? "" : "!"), (p->precious ? "" : "!")); iprintf("%sabsent, %serror, %sdirty, %scleaning, %spageout, %sclustered\n", (p->absent ? "" : "!"), (p->error ? "" : "!"), (p->dirty ? "" : "!"), (p->cleaning ? "" : "!"), (p->pageout ? "" : "!"), (p->clustered ? "" : "!")); iprintf("%slock_supplied, %soverwriting, %srestart, %sunusual, %slimbo\n", (p->lock_supplied ? "" : "!"), (p->overwriting ? "" : "!"), (p->restart ? "" : "!"), (p->unusual ? "" : "!"), (p->limbo ? "" : "!")); iprintf("phys_addr=0x%x", p->phys_addr); printf(", page_error=0x%x", p->page_error); printf(", page_lock=0x%x", p->page_lock); printf(", unlock_request=%d\n", p->unlock_request); db_indent -= 2; } #endif /* MACH_KDB */