gzalloc.c
1 /* 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * File: kern/gzalloc.c 30 * Author: Derek Kumar 31 * 32 * "Guard mode" zone allocator, used to trap use-after-free errors, 33 * overruns, underruns, mismatched allocations/frees, uninitialized 34 * zone element use, timing dependent races etc. 35 * 36 * The allocator is configured by these boot-args: 37 * gzalloc_size=<size>: target all zones with elements of <size> bytes 38 * gzalloc_min=<size>: target zones with elements >= size 39 * gzalloc_max=<size>: target zones with elements <= size 40 * gzalloc_min/max can be specified in conjunction to target a range of 41 * sizes 42 * gzalloc_fc_size=<size>: number of zone elements (effectively page 43 * multiple sized) to retain in the free VA cache. This cache is evicted 44 * (backing pages and VA released) in a least-recently-freed fashion. 45 * Larger free VA caches allow for a longer window of opportunity to trap 46 * delayed use-after-free operations, but use more memory. 47 * -gzalloc_wp: Write protect, rather than unmap, freed allocations 48 * lingering in the free VA cache. Useful to disambiguate between 49 * read-after-frees/read overruns and writes. Also permits direct inspection 50 * of the freed element in the cache via the kernel debugger. As each 51 * element has a "header" (trailer in underflow detection mode), the zone 52 * of origin of the element can be easily determined in this mode. 53 * -gzalloc_uf_mode: Underflow detection mode, where the guard page 54 * adjoining each element is placed *before* the element page rather than 55 * after. The element is also located at the top of the page, rather than 56 * abutting the bottom as with the standard overflow detection mode. 57 * -gzalloc_noconsistency: disable consistency checks that flag mismatched 58 * frees, corruptions of the header/trailer signatures etc. 59 * -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel 60 * enables the guard allocator for zones sized 1K (if present) by 61 * default, this option can disable that behaviour. 62 * gzname=<name> target a zone by name. Can be coupled with size-based 63 * targeting. Naming conventions match those of the zlog boot-arg, i.e. 64 * "a period in the logname will match a space in the zone name" 65 * -gzalloc_no_dfree_check Eliminate double free checks 66 * gzalloc_zscale=<value> specify size multiplier for the dedicated gzalloc submap 67 */ 68 69 #include <mach/mach_types.h> 70 #include <mach/vm_param.h> 71 #include <mach/kern_return.h> 72 #include <mach/machine/vm_types.h> 73 #include <mach_debug/zone_info.h> 74 #include <mach/vm_map.h> 75 76 #include <kern/kern_types.h> 77 #include <kern/assert.h> 78 #include <kern/sched.h> 79 #include <kern/locks.h> 80 #include <kern/misc_protos.h> 81 #include <kern/zalloc_internal.h> 82 83 #include <vm/pmap.h> 84 #include <vm/vm_map.h> 85 #include <vm/vm_kern.h> 86 #include <vm/vm_page.h> 87 88 #include <pexpert/pexpert.h> 89 90 #include <machine/machparam.h> 91 92 #include <libkern/OSDebug.h> 93 #include <libkern/OSAtomic.h> 94 #include <sys/kdebug.h> 95 96 boolean_t gzalloc_mode = FALSE; 97 uint32_t pdzalloc_count, pdzfree_count; 98 99 #define GZALLOC_MIN_DEFAULT (1024) 100 #define GZDEADZONE ((zone_t) 0xDEAD201E) 101 #define GZALLOC_SIGNATURE (0xABADCAFE) 102 #define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024) 103 #define GZFC_DEFAULT_SIZE (1536) 104 105 char gzalloc_fill_pattern = 0x67; /* 'g' */ 106 107 uint32_t gzalloc_min = ~0U; 108 uint32_t gzalloc_max = 0; 109 uint32_t gzalloc_size = 0; 110 uint64_t gzalloc_allocated, gzalloc_freed, gzalloc_early_alloc, gzalloc_early_free, gzalloc_wasted; 111 boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE, gzalloc_dfree_check = TRUE; 112 vm_prot_t gzalloc_prot = VM_PROT_NONE; 113 uint32_t gzalloc_guard = KMA_GUARD_LAST; 114 uint32_t gzfc_size = GZFC_DEFAULT_SIZE; 115 uint32_t gzalloc_zonemap_scale = 6; 116 117 vm_map_t gzalloc_map; 118 vm_offset_t gzalloc_map_min, gzalloc_map_max; 119 vm_offset_t gzalloc_reserve; 120 vm_size_t gzalloc_reserve_size; 121 122 typedef struct gzalloc_header { 123 zone_t gzone; 124 uint32_t gzsize; 125 uint32_t gzsig; 126 } gzhdr_t; 127 128 #define GZHEADER_SIZE (sizeof(gzhdr_t)) 129 130 extern zone_t vm_page_zone; 131 132 static zone_t gztrackzone = NULL; 133 static char gznamedzone[MAX_ZONE_NAME] = ""; 134 135 boolean_t 136 gzalloc_enabled(void) 137 { 138 return gzalloc_mode; 139 } 140 141 void 142 gzalloc_zone_init(zone_t z) 143 { 144 if (gzalloc_mode == 0) { 145 return; 146 } 147 148 bzero(&z->gz, sizeof(z->gz)); 149 150 if (track_this_zone(z->z_name, gznamedzone)) { 151 gztrackzone = z; 152 } 153 154 if (!z->gzalloc_exempt) { 155 z->gzalloc_tracked = (z == gztrackzone) || 156 ((zone_elem_size(z) >= gzalloc_min) && (zone_elem_size(z) <= gzalloc_max)); 157 } 158 159 if (gzfc_size && z->gzalloc_tracked) { 160 vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size); 161 kern_return_t kr; 162 163 /* If the VM/kmem system aren't yet configured, carve 164 * out the free element cache structure directly from the 165 * gzalloc_reserve supplied by the pmap layer. 166 */ 167 if (__improbable(startup_phase < STARTUP_SUB_KMEM)) { 168 if (gzalloc_reserve_size < gzfcsz) { 169 panic("gzalloc reserve exhausted"); 170 } 171 172 z->gz.gzfc = (vm_offset_t *)gzalloc_reserve; 173 gzalloc_reserve += gzfcsz; 174 gzalloc_reserve_size -= gzfcsz; 175 bzero(z->gz.gzfc, gzfcsz); 176 } else { 177 kr = kernel_memory_allocate(kernel_map, 178 (vm_offset_t *)&z->gz.gzfc, gzfcsz, 0, 179 KMA_KOBJECT | KMA_ZERO, VM_KERN_MEMORY_OSFMK); 180 if (kr != KERN_SUCCESS) { 181 panic("%s: kernel_memory_allocate failed (%d) for 0x%lx bytes", 182 __func__, kr, (unsigned long)gzfcsz); 183 } 184 } 185 } 186 } 187 188 /* Called by zdestroy() to dump the free cache elements so the zone count can drop to zero. */ 189 void 190 gzalloc_empty_free_cache(zone_t zone) 191 { 192 kern_return_t kr; 193 int freed_elements = 0; 194 vm_offset_t free_addr = 0; 195 vm_offset_t rounded_size = round_page(zone_elem_size(zone) + GZHEADER_SIZE); 196 vm_offset_t gzfcsz = round_page(sizeof(*zone->gz.gzfc) * gzfc_size); 197 vm_offset_t gzfc_copy; 198 199 assert(zone->gzalloc_tracked); // the caller is responsible for checking 200 201 kr = kmem_alloc(kernel_map, &gzfc_copy, gzfcsz, VM_KERN_MEMORY_OSFMK); 202 if (kr != KERN_SUCCESS) { 203 panic("gzalloc_empty_free_cache: kmem_alloc: 0x%x", kr); 204 } 205 206 /* Reset gzalloc_data. */ 207 zone_lock(zone); 208 memcpy((void *)gzfc_copy, (void *)zone->gz.gzfc, gzfcsz); 209 bzero((void *)zone->gz.gzfc, gzfcsz); 210 zone->gz.gzfc_index = 0; 211 zone_unlock(zone); 212 213 /* Free up all the cached elements. */ 214 for (uint32_t index = 0; index < gzfc_size; index++) { 215 free_addr = ((vm_offset_t *)gzfc_copy)[index]; 216 if (free_addr && free_addr >= gzalloc_map_min && free_addr < gzalloc_map_max) { 217 kr = vm_map_remove(gzalloc_map, free_addr, 218 free_addr + rounded_size + (1 * PAGE_SIZE), 219 VM_MAP_REMOVE_KUNWIRE); 220 if (kr != KERN_SUCCESS) { 221 panic("gzalloc_empty_free_cache: vm_map_remove: %p, 0x%x", (void *)free_addr, kr); 222 } 223 OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed); 224 OSAddAtomic64(-((SInt32) (rounded_size - zone_elem_size(zone))), &gzalloc_wasted); 225 226 freed_elements++; 227 } 228 } 229 /* 230 * TODO: Consider freeing up zone->gz.gzfc as well if it didn't come from the gzalloc_reserve pool. 231 * For now we're reusing this buffer across zdestroy's. We would have to allocate it again on a 232 * subsequent zinit() as well. 233 */ 234 235 /* Decrement zone counters. */ 236 zone_lock(zone); 237 zone->z_elems_free += freed_elements; 238 zone->z_wired_cur -= freed_elements; 239 zone_unlock(zone); 240 241 kmem_free(kernel_map, gzfc_copy, gzfcsz); 242 } 243 244 __startup_func 245 static void 246 gzalloc_configure(void) 247 { 248 #if !KASAN_ZALLOC 249 char temp_buf[16]; 250 251 if (PE_parse_boot_argn("-gzalloc_mode", temp_buf, sizeof(temp_buf))) { 252 gzalloc_mode = TRUE; 253 gzalloc_min = GZALLOC_MIN_DEFAULT; 254 gzalloc_max = ~0U; 255 } 256 257 if (PE_parse_boot_argn("gzalloc_min", &gzalloc_min, sizeof(gzalloc_min))) { 258 gzalloc_mode = TRUE; 259 gzalloc_max = ~0U; 260 } 261 262 if (PE_parse_boot_argn("gzalloc_max", &gzalloc_max, sizeof(gzalloc_max))) { 263 gzalloc_mode = TRUE; 264 if (gzalloc_min == ~0U) { 265 gzalloc_min = 0; 266 } 267 } 268 269 if (PE_parse_boot_argn("gzalloc_size", &gzalloc_size, sizeof(gzalloc_size))) { 270 gzalloc_min = gzalloc_max = gzalloc_size; 271 gzalloc_mode = TRUE; 272 } 273 274 (void)PE_parse_boot_argn("gzalloc_fc_size", &gzfc_size, sizeof(gzfc_size)); 275 276 if (PE_parse_boot_argn("-gzalloc_wp", temp_buf, sizeof(temp_buf))) { 277 gzalloc_prot = VM_PROT_READ; 278 } 279 280 if (PE_parse_boot_argn("-gzalloc_uf_mode", temp_buf, sizeof(temp_buf))) { 281 gzalloc_uf_mode = TRUE; 282 gzalloc_guard = KMA_GUARD_FIRST; 283 } 284 285 if (PE_parse_boot_argn("-gzalloc_no_dfree_check", temp_buf, sizeof(temp_buf))) { 286 gzalloc_dfree_check = FALSE; 287 } 288 289 (void) PE_parse_boot_argn("gzalloc_zscale", &gzalloc_zonemap_scale, sizeof(gzalloc_zonemap_scale)); 290 291 if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof(temp_buf))) { 292 gzalloc_consistency_checks = FALSE; 293 } 294 295 if (PE_parse_boot_argn("gzname", gznamedzone, sizeof(gznamedzone))) { 296 gzalloc_mode = TRUE; 297 } 298 #if DEBUG 299 if (gzalloc_mode == FALSE) { 300 gzalloc_min = 1024; 301 gzalloc_max = 1024; 302 strlcpy(gznamedzone, "pmap", sizeof(gznamedzone)); 303 gzalloc_prot = VM_PROT_READ; 304 gzalloc_mode = TRUE; 305 } 306 #endif 307 if (PE_parse_boot_argn("-nogzalloc_mode", temp_buf, sizeof(temp_buf))) { 308 gzalloc_mode = FALSE; 309 } 310 311 if (gzalloc_mode) { 312 gzalloc_reserve_size = GZALLOC_RESERVE_SIZE_DEFAULT; 313 gzalloc_reserve = (vm_offset_t) pmap_steal_memory(gzalloc_reserve_size); 314 } 315 #endif 316 } 317 STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, gzalloc_configure); 318 319 void 320 gzalloc_init(vm_size_t max_zonemap_size) 321 { 322 kern_return_t retval; 323 324 if (gzalloc_mode) { 325 vm_map_kernel_flags_t vmk_flags; 326 327 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; 328 vmk_flags.vmkf_permanent = TRUE; 329 retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size * gzalloc_zonemap_scale), 330 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE, 331 &gzalloc_map); 332 333 if (retval != KERN_SUCCESS) { 334 panic("zone_init: kmem_suballoc(gzalloc_map, 0x%lx, %u) failed", 335 max_zonemap_size, gzalloc_zonemap_scale); 336 } 337 gzalloc_map_max = gzalloc_map_min + (max_zonemap_size * gzalloc_zonemap_scale); 338 } 339 } 340 341 vm_offset_t 342 gzalloc_alloc(zone_t zone, zone_stats_t zstats, zalloc_flags_t flags) 343 { 344 vm_offset_t addr = 0; 345 346 assert(zone->gzalloc_tracked); // the caller is responsible for checking 347 348 if (get_preemption_level() != 0) { 349 if (flags & Z_NOWAIT) { 350 return 0; 351 } 352 pdzalloc_count++; 353 } 354 355 bool kmem_ready = (startup_phase >= STARTUP_SUB_KMEM); 356 vm_offset_t rounded_size = round_page(zone_elem_size(zone) + GZHEADER_SIZE); 357 vm_offset_t residue = rounded_size - zone_elem_size(zone); 358 vm_offset_t gzaddr = 0; 359 gzhdr_t *gzh, *gzhcopy = NULL; 360 bool new_va = false; 361 362 if (!kmem_ready || (vm_page_zone == ZONE_NULL)) { 363 /* Early allocations are supplied directly from the 364 * reserve. 365 */ 366 if (gzalloc_reserve_size < (rounded_size + PAGE_SIZE)) { 367 panic("gzalloc reserve exhausted"); 368 } 369 gzaddr = gzalloc_reserve; 370 /* No guard page for these early allocations, just 371 * waste an additional page. 372 */ 373 gzalloc_reserve += rounded_size + PAGE_SIZE; 374 gzalloc_reserve_size -= rounded_size + PAGE_SIZE; 375 OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_alloc); 376 } else { 377 kern_return_t kr = kernel_memory_allocate(gzalloc_map, 378 &gzaddr, rounded_size + (1 * PAGE_SIZE), 379 0, KMA_KOBJECT | KMA_ATOMIC | gzalloc_guard, 380 VM_KERN_MEMORY_OSFMK); 381 if (kr != KERN_SUCCESS) { 382 panic("gzalloc: kernel_memory_allocate for size 0x%llx failed with %d", 383 (uint64_t)rounded_size, kr); 384 } 385 new_va = true; 386 } 387 388 if (gzalloc_uf_mode) { 389 gzaddr += PAGE_SIZE; 390 /* The "header" becomes a "footer" in underflow 391 * mode. 392 */ 393 gzh = (gzhdr_t *) (gzaddr + zone_elem_size(zone)); 394 addr = gzaddr; 395 gzhcopy = (gzhdr_t *) (gzaddr + rounded_size - sizeof(gzhdr_t)); 396 } else { 397 gzh = (gzhdr_t *) (gzaddr + residue - GZHEADER_SIZE); 398 addr = (gzaddr + residue); 399 } 400 401 if (zone->z_free_zeroes) { 402 bzero((void *)gzaddr, rounded_size); 403 } else { 404 /* Fill with a pattern on allocation to trap uninitialized 405 * data use. Since the element size may be "rounded up" 406 * by higher layers such as the kalloc layer, this may 407 * also identify overruns between the originally requested 408 * size and the rounded size via visual inspection. 409 * TBD: plumb through the originally requested size, 410 * prior to rounding by kalloc/IOMalloc etc. 411 * We also add a signature and the zone of origin in a header 412 * prefixed to the allocation. 413 */ 414 memset((void *)gzaddr, gzalloc_fill_pattern, rounded_size); 415 } 416 417 gzh->gzone = (kmem_ready && vm_page_zone) ? zone : GZDEADZONE; 418 gzh->gzsize = (uint32_t)zone_elem_size(zone); 419 gzh->gzsig = GZALLOC_SIGNATURE; 420 421 /* In underflow detection mode, stash away a copy of the 422 * metadata at the edge of the allocated range, for 423 * retrieval by gzalloc_element_size() 424 */ 425 if (gzhcopy) { 426 *gzhcopy = *gzh; 427 } 428 429 zone_lock(zone); 430 assert(zone->z_self == zone); 431 zone->z_elems_free--; 432 if (new_va) { 433 zone->z_va_cur += 1; 434 } 435 zone->z_wired_cur += 1; 436 zpercpu_get(zstats)->zs_mem_allocated += rounded_size; 437 zone_unlock(zone); 438 439 OSAddAtomic64((SInt32) rounded_size, &gzalloc_allocated); 440 OSAddAtomic64((SInt32) (rounded_size - zone_elem_size(zone)), &gzalloc_wasted); 441 442 return addr; 443 } 444 445 void 446 gzalloc_free(zone_t zone, zone_stats_t zstats, void *addr) 447 { 448 kern_return_t kr; 449 450 assert(zone->gzalloc_tracked); // the caller is responsible for checking 451 452 gzhdr_t *gzh; 453 vm_offset_t rounded_size = round_page(zone_elem_size(zone) + GZHEADER_SIZE); 454 vm_offset_t residue = rounded_size - zone_elem_size(zone); 455 vm_offset_t saddr; 456 vm_offset_t free_addr = 0; 457 458 if (gzalloc_uf_mode) { 459 gzh = (gzhdr_t *)((vm_offset_t)addr + zone_elem_size(zone)); 460 saddr = (vm_offset_t) addr - PAGE_SIZE; 461 } else { 462 gzh = (gzhdr_t *)((vm_offset_t)addr - GZHEADER_SIZE); 463 saddr = ((vm_offset_t)addr) - residue; 464 } 465 466 if ((saddr & PAGE_MASK) != 0) { 467 panic("%s: invalid address supplied: " 468 "%p (adjusted: 0x%lx) for zone with element sized 0x%lx\n", 469 __func__, addr, saddr, zone_elem_size(zone)); 470 } 471 472 if (gzfc_size && gzalloc_dfree_check) { 473 zone_lock(zone); 474 assert(zone->z_self == zone); 475 for (uint32_t gd = 0; gd < gzfc_size; gd++) { 476 if (zone->gz.gzfc[gd] != saddr) { 477 continue; 478 } 479 panic("%s: double free detected, freed address: 0x%lx, " 480 "current free cache index: %d, freed index: %d", 481 __func__, saddr, zone->gz.gzfc_index, gd); 482 } 483 zone_unlock(zone); 484 } 485 486 if (gzalloc_consistency_checks) { 487 if (gzh->gzsig != GZALLOC_SIGNATURE) { 488 panic("GZALLOC signature mismatch for element %p, " 489 "expected 0x%x, found 0x%x", 490 addr, GZALLOC_SIGNATURE, gzh->gzsig); 491 } 492 493 if (gzh->gzone != zone && (gzh->gzone != GZDEADZONE)) { 494 panic("%s: Mismatched zone or under/overflow, " 495 "current zone: %p, recorded zone: %p, address: %p", 496 __func__, zone, gzh->gzone, (void *)addr); 497 } 498 /* Partially redundant given the zone check, but may flag header corruption */ 499 if (gzh->gzsize != zone_elem_size(zone)) { 500 panic("Mismatched zfree or under/overflow for zone %p, " 501 "recorded size: 0x%x, element size: 0x%x, address: %p", 502 zone, gzh->gzsize, (uint32_t)zone_elem_size(zone), (void *)addr); 503 } 504 505 char *gzc, *checkstart, *checkend; 506 if (gzalloc_uf_mode) { 507 checkstart = (char *) ((uintptr_t) gzh + sizeof(gzh)); 508 checkend = (char *) ((((vm_offset_t)addr) & ~PAGE_MASK) + PAGE_SIZE); 509 } else { 510 checkstart = (char *) trunc_page_64(addr); 511 checkend = (char *)gzh; 512 } 513 514 for (gzc = checkstart; gzc < checkend; gzc++) { 515 if (*gzc == gzalloc_fill_pattern) { 516 continue; 517 } 518 panic("%s: detected over/underflow, byte at %p, element %p, " 519 "contents 0x%x from 0x%lx byte sized zone (%s%s) " 520 "doesn't match fill pattern (%c)", 521 __func__, gzc, addr, *gzc, zone_elem_size(zone), 522 zone_heap_name(zone), zone->z_name, gzalloc_fill_pattern); 523 } 524 } 525 526 if ((startup_phase < STARTUP_SUB_KMEM) || gzh->gzone == GZDEADZONE) { 527 /* For now, just leak frees of early allocations 528 * performed before kmem is fully configured. 529 * They don't seem to get freed currently; 530 * consider ml_static_mfree in the future. 531 */ 532 OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_free); 533 return; 534 } 535 536 if (get_preemption_level() != 0) { 537 pdzfree_count++; 538 } 539 540 if (gzfc_size) { 541 /* Either write protect or unmap the newly freed 542 * allocation 543 */ 544 kr = vm_map_protect(gzalloc_map, saddr, 545 saddr + rounded_size + (1 * PAGE_SIZE), 546 gzalloc_prot, FALSE); 547 if (kr != KERN_SUCCESS) { 548 panic("%s: vm_map_protect: %p, 0x%x", __func__, (void *)saddr, kr); 549 } 550 } else { 551 free_addr = saddr; 552 } 553 554 zone_lock(zone); 555 assert(zone->z_self == zone); 556 557 /* Insert newly freed element into the protected free element 558 * cache, and rotate out the LRU element. 559 */ 560 if (gzfc_size) { 561 if (zone->gz.gzfc_index >= gzfc_size) { 562 zone->gz.gzfc_index = 0; 563 } 564 free_addr = zone->gz.gzfc[zone->gz.gzfc_index]; 565 zone->gz.gzfc[zone->gz.gzfc_index++] = saddr; 566 } 567 568 if (free_addr) { 569 zone->z_elems_free++; 570 zone->z_wired_cur -= 1; 571 } 572 573 zpercpu_get(zstats)->zs_mem_freed += rounded_size; 574 zone_unlock(zone); 575 576 if (free_addr) { 577 // TODO: consider using physical reads to check for 578 // corruption while on the protected freelist 579 // (i.e. physical corruption) 580 kr = vm_map_remove(gzalloc_map, free_addr, 581 free_addr + rounded_size + (1 * PAGE_SIZE), 582 VM_MAP_REMOVE_KUNWIRE); 583 if (kr != KERN_SUCCESS) { 584 panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr); 585 } 586 // TODO: sysctl-ize for quick reference 587 OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed); 588 OSAddAtomic64(-((SInt32) (rounded_size - zone_elem_size(zone))), 589 &gzalloc_wasted); 590 } 591 } 592 593 boolean_t 594 gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz) 595 { 596 uintptr_t a = (uintptr_t)gzaddr; 597 if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a < gzalloc_map_max))) { 598 gzhdr_t *gzh; 599 boolean_t vmef; 600 vm_map_entry_t gzvme = NULL; 601 vm_map_lock_read(gzalloc_map); 602 vmef = vm_map_lookup_entry(gzalloc_map, (vm_map_offset_t)a, &gzvme); 603 vm_map_unlock(gzalloc_map); 604 if (vmef == FALSE) { 605 panic("GZALLOC: unable to locate map entry for %p\n", (void *)a); 606 } 607 assertf(gzvme->vme_atomic != 0, "GZALLOC: VM map entry inconsistency, " 608 "vme: %p, start: %llu end: %llu", gzvme, gzvme->vme_start, gzvme->vme_end); 609 610 /* Locate the gzalloc metadata adjoining the element */ 611 if (gzalloc_uf_mode == TRUE) { 612 /* In underflow detection mode, locate the map entry describing 613 * the element, and then locate the copy of the gzalloc 614 * header at the trailing edge of the range. 615 */ 616 gzh = (gzhdr_t *)(gzvme->vme_end - GZHEADER_SIZE); 617 } else { 618 /* In overflow detection mode, scan forward from 619 * the base of the map entry to locate the 620 * gzalloc header. 621 */ 622 uint32_t *p = (uint32_t*) gzvme->vme_start; 623 while (p < (uint32_t *) gzvme->vme_end) { 624 if (*p == GZALLOC_SIGNATURE) { 625 break; 626 } else { 627 p++; 628 } 629 } 630 if (p >= (uint32_t *) gzvme->vme_end) { 631 panic("GZALLOC signature missing addr %p, zone %p", gzaddr, z); 632 } 633 p++; 634 uintptr_t q = (uintptr_t) p; 635 gzh = (gzhdr_t *) (q - sizeof(gzhdr_t)); 636 } 637 638 if (gzh->gzsig != GZALLOC_SIGNATURE) { 639 panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", 640 (void *)a, GZALLOC_SIGNATURE, gzh->gzsig); 641 } 642 643 *gzsz = zone_elem_size(gzh->gzone); 644 if (__improbable(!gzh->gzone->gzalloc_tracked)) { 645 panic("GZALLOC: zone mismatch (%p)\n", gzh->gzone); 646 } 647 648 if (z) { 649 *z = gzh->gzone; 650 } 651 return TRUE; 652 } else { 653 return FALSE; 654 } 655 }