kalloc.c
1 /* 2 * Copyright (c) 2000-2020 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 /* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 /* 57 */ 58 /* 59 * File: kern/kalloc.c 60 * Author: Avadis Tevanian, Jr. 61 * Date: 1985 62 * 63 * General kernel memory allocator. This allocator is designed 64 * to be used by the kernel to manage dynamic memory fast. 65 */ 66 67 #include <mach/boolean.h> 68 #include <mach/sdt.h> 69 #include <mach/machine/vm_types.h> 70 #include <mach/vm_param.h> 71 #include <kern/misc_protos.h> 72 #include <kern/zalloc_internal.h> 73 #include <kern/kalloc.h> 74 #include <kern/ledger.h> 75 #include <kern/backtrace.h> 76 #include <vm/vm_kern.h> 77 #include <vm/vm_object.h> 78 #include <vm/vm_map.h> 79 #include <sys/kdebug.h> 80 81 #include <san/kasan.h> 82 #include <libkern/section_keywords.h> 83 84 /* #define KALLOC_DEBUG 1 */ 85 86 #define KALLOC_MAP_SIZE_MIN (16 * 1024 * 1024) 87 #define KALLOC_MAP_SIZE_MAX (128 * 1024 * 1024) 88 89 static SECURITY_READ_ONLY_LATE(vm_offset_t) kalloc_map_min; 90 static SECURITY_READ_ONLY_LATE(vm_offset_t) kalloc_map_max; 91 static SECURITY_READ_ONLY_LATE(vm_size_t) kalloc_max; 92 SECURITY_READ_ONLY_LATE(vm_size_t) kalloc_max_prerounded; 93 /* size of kallocs that can come from kernel map */ 94 SECURITY_READ_ONLY_LATE(vm_size_t) kalloc_kernmap_size; 95 SECURITY_READ_ONLY_LATE(vm_map_t) kalloc_map; 96 #if DEBUG || DEVELOPMENT 97 static TUNABLE(bool, kheap_temp_debug, "kheap_temp_debug", false); 98 99 #define KHT_BT_COUNT 14 100 struct kheap_temp_header { 101 queue_chain_t kht_hdr_link; 102 uintptr_t kht_hdr_pcs[KHT_BT_COUNT]; 103 }; 104 #endif 105 106 /* how many times we couldn't allocate out of kalloc_map and fell back to kernel_map */ 107 unsigned long kalloc_fallback_count; 108 109 uint_t kalloc_large_inuse; 110 vm_size_t kalloc_large_total; 111 vm_size_t kalloc_large_max; 112 vm_size_t kalloc_largest_allocated = 0; 113 uint64_t kalloc_large_sum; 114 115 LCK_GRP_DECLARE(kalloc_lck_grp, "kalloc.large"); 116 LCK_SPIN_DECLARE(kalloc_lock, &kalloc_lck_grp); 117 118 #define kalloc_spin_lock() lck_spin_lock(&kalloc_lock) 119 #define kalloc_unlock() lck_spin_unlock(&kalloc_lock) 120 121 #pragma mark initialization 122 123 /* 124 * All allocations of size less than kalloc_max are rounded to the next nearest 125 * sized zone. This allocator is built on top of the zone allocator. A zone 126 * is created for each potential size that we are willing to get in small 127 * blocks. 128 * 129 * We assume that kalloc_max is not greater than 64K; 130 * 131 * Note that kalloc_max is somewhat confusingly named. It represents the first 132 * power of two for which no zone exists. kalloc_max_prerounded is the 133 * smallest allocation size, before rounding, for which no zone exists. 134 * 135 * Also if the allocation size is more than kalloc_kernmap_size then allocate 136 * from kernel map rather than kalloc_map. 137 */ 138 139 #define KiB(x) (1024 * (x)) 140 141 /* 142 * The k_zone_cfg table defines the configuration of zones on various platforms. 143 * The currently defined list of zones and their per-CPU caching behavior are as 144 * follows 145 * 146 * X:zone not present 147 * N:zone present no cpu-caching 148 * Y:zone present with cpu-caching 149 * 150 * Size macOS(64-bit) embedded(32-bit) embedded(64-bit) 151 *-------- ---------------- ---------------- ---------------- 152 * 153 * 8 X Y X 154 * 16 Y Y Y 155 * 24 X Y X 156 * 32 Y Y Y 157 * 40 X Y X 158 * 48 Y Y Y 159 * 64 Y Y Y 160 * 72 X Y X 161 * 80 Y X Y 162 * 88 X Y X 163 * 96 Y X Y 164 * 112 X Y X 165 * 128 Y Y Y 166 * 160 Y X Y 167 * 192 Y Y Y 168 * 224 Y X Y 169 * 256 Y Y Y 170 * 288 Y Y Y 171 * 368 Y X Y 172 * 384 X Y X 173 * 400 Y X Y 174 * 440 X Y X 175 * 512 Y Y Y 176 * 576 Y N N 177 * 768 Y N N 178 * 1024 Y Y Y 179 * 1152 N N N 180 * 1280 N N N 181 * 1536 X N X 182 * 1664 N X N 183 * 2048 Y N N 184 * 2128 X N X 185 * 3072 X N X 186 * 4096 Y N N 187 * 6144 N N N 188 * 8192 Y N N 189 * 12288 N X X 190 * 16384 N X N 191 * 32768 X X N 192 * 193 */ 194 struct kalloc_zone_cfg { 195 bool kzc_caching; 196 uint32_t kzc_size; 197 const char *kzc_name; 198 }; 199 static SECURITY_READ_ONLY_LATE(struct kalloc_zone_cfg) k_zone_cfg[] = { 200 #define KZC_ENTRY(SIZE, caching) { \ 201 .kzc_caching = (caching), \ 202 .kzc_size = (SIZE), \ 203 .kzc_name = "kalloc." #SIZE \ 204 } 205 206 #if !defined(XNU_TARGET_OS_OSX) 207 208 #if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4 209 /* Zone config for embedded 64-bit platforms */ 210 KZC_ENTRY(16, true), 211 KZC_ENTRY(32, true), 212 KZC_ENTRY(48, true), 213 KZC_ENTRY(64, true), 214 KZC_ENTRY(80, true), 215 KZC_ENTRY(96, true), 216 KZC_ENTRY(128, true), 217 KZC_ENTRY(160, true), 218 KZC_ENTRY(192, true), 219 KZC_ENTRY(224, true), 220 KZC_ENTRY(256, true), 221 KZC_ENTRY(288, true), 222 KZC_ENTRY(368, true), 223 KZC_ENTRY(400, true), 224 KZC_ENTRY(512, true), 225 KZC_ENTRY(576, false), 226 KZC_ENTRY(768, false), 227 KZC_ENTRY(1024, true), 228 KZC_ENTRY(1152, false), 229 KZC_ENTRY(1280, false), 230 KZC_ENTRY(1664, false), 231 KZC_ENTRY(2048, false), 232 KZC_ENTRY(4096, false), 233 KZC_ENTRY(6144, false), 234 KZC_ENTRY(8192, false), 235 KZC_ENTRY(16384, false), 236 KZC_ENTRY(32768, false), 237 238 #elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3 239 /* Zone config for embedded 32-bit platforms */ 240 KZC_ENTRY(8, true), 241 KZC_ENTRY(16, true), 242 KZC_ENTRY(24, true), 243 KZC_ENTRY(32, true), 244 KZC_ENTRY(40, true), 245 KZC_ENTRY(48, true), 246 KZC_ENTRY(64, true), 247 KZC_ENTRY(72, true), 248 KZC_ENTRY(88, true), 249 KZC_ENTRY(112, true), 250 KZC_ENTRY(128, true), 251 KZC_ENTRY(192, true), 252 KZC_ENTRY(256, true), 253 KZC_ENTRY(288, true), 254 KZC_ENTRY(384, true), 255 KZC_ENTRY(440, true), 256 KZC_ENTRY(512, true), 257 KZC_ENTRY(576, false), 258 KZC_ENTRY(768, false), 259 KZC_ENTRY(1024, true), 260 KZC_ENTRY(1152, false), 261 KZC_ENTRY(1280, false), 262 KZC_ENTRY(1536, false), 263 KZC_ENTRY(2048, false), 264 KZC_ENTRY(2128, false), 265 KZC_ENTRY(3072, false), 266 KZC_ENTRY(4096, false), 267 KZC_ENTRY(6144, false), 268 KZC_ENTRY(8192, false), 269 /* To limit internal fragmentation, only add the following zones if the 270 * page size is greater than 4K. 271 * Note that we use ARM_PGBYTES here (instead of one of the VM macros) 272 * since it's guaranteed to be a compile time constant. 273 */ 274 #if ARM_PGBYTES > 4096 275 KZC_ENTRY(16384, false), 276 KZC_ENTRY(32768, false), 277 #endif /* ARM_PGBYTES > 4096 */ 278 279 #else 280 #error missing or invalid zone size parameters for kalloc 281 #endif 282 283 #else /* !defined(XNU_TARGET_OS_OSX) */ 284 285 /* Zone config for macOS 64-bit platforms */ 286 KZC_ENTRY(16, true), 287 KZC_ENTRY(32, true), 288 KZC_ENTRY(48, true), 289 KZC_ENTRY(64, true), 290 KZC_ENTRY(80, true), 291 KZC_ENTRY(96, true), 292 KZC_ENTRY(128, true), 293 KZC_ENTRY(160, true), 294 KZC_ENTRY(192, true), 295 KZC_ENTRY(224, true), 296 KZC_ENTRY(256, true), 297 KZC_ENTRY(288, true), 298 KZC_ENTRY(368, true), 299 KZC_ENTRY(400, true), 300 KZC_ENTRY(512, true), 301 KZC_ENTRY(576, true), 302 KZC_ENTRY(768, true), 303 KZC_ENTRY(1024, true), 304 KZC_ENTRY(1152, false), 305 KZC_ENTRY(1280, false), 306 KZC_ENTRY(1664, false), 307 KZC_ENTRY(2048, true), 308 KZC_ENTRY(4096, true), 309 KZC_ENTRY(6144, false), 310 KZC_ENTRY(8192, true), 311 KZC_ENTRY(12288, false), 312 KZC_ENTRY(16384, false) 313 314 #endif /* !defined(XNU_TARGET_OS_OSX) */ 315 316 #undef KZC_ENTRY 317 }; 318 319 #define MAX_K_ZONE(kzc) (uint32_t)(sizeof(kzc) / sizeof(kzc[0])) 320 321 /* 322 * Many kalloc() allocations are for small structures containing a few 323 * pointers and longs - the dlut[] direct lookup table, indexed by 324 * size normalized to the minimum alignment, finds the right zone index 325 * for them in one dereference. 326 */ 327 328 #define INDEX_ZDLUT(size) (((size) + KALLOC_MINALIGN - 1) / KALLOC_MINALIGN) 329 #define MAX_SIZE_ZDLUT ((KALLOC_DLUT_SIZE - 1) * KALLOC_MINALIGN) 330 331 static SECURITY_READ_ONLY_LATE(zone_t) k_zone_default[MAX_K_ZONE(k_zone_cfg)]; 332 static SECURITY_READ_ONLY_LATE(zone_t) k_zone_data_buffers[MAX_K_ZONE(k_zone_cfg)]; 333 static SECURITY_READ_ONLY_LATE(zone_t) k_zone_kext[MAX_K_ZONE(k_zone_cfg)]; 334 335 #if VM_MAX_TAG_ZONES 336 #if __LP64__ 337 static_assert(VM_MAX_TAG_ZONES >= 338 MAX_K_ZONE(k_zone_cfg) + MAX_K_ZONE(k_zone_cfg) + MAX_K_ZONE(k_zone_cfg)); 339 #else 340 static_assert(VM_MAX_TAG_ZONES >= MAX_K_ZONE(k_zone_cfg)); 341 #endif 342 #endif 343 344 const char * const kalloc_heap_names[] = { 345 [KHEAP_ID_NONE] = "", 346 [KHEAP_ID_DEFAULT] = "default.", 347 [KHEAP_ID_DATA_BUFFERS] = "data.", 348 [KHEAP_ID_KEXT] = "kext.", 349 }; 350 351 /* 352 * Default kalloc heap configuration 353 */ 354 static SECURITY_READ_ONLY_LATE(struct kheap_zones) kalloc_zones_default = { 355 .cfg = k_zone_cfg, 356 .heap_id = KHEAP_ID_DEFAULT, 357 .k_zone = k_zone_default, 358 .max_k_zone = MAX_K_ZONE(k_zone_cfg) 359 }; 360 SECURITY_READ_ONLY_LATE(struct kalloc_heap) KHEAP_DEFAULT[1] = { 361 { 362 .kh_zones = &kalloc_zones_default, 363 .kh_name = "default.", 364 .kh_heap_id = KHEAP_ID_DEFAULT, 365 } 366 }; 367 368 KALLOC_HEAP_DEFINE(KHEAP_TEMP, "temp allocations", KHEAP_ID_DEFAULT); 369 370 371 /* 372 * Bag of bytes heap configuration 373 */ 374 static SECURITY_READ_ONLY_LATE(struct kheap_zones) kalloc_zones_data_buffers = { 375 .cfg = k_zone_cfg, 376 .heap_id = KHEAP_ID_DATA_BUFFERS, 377 .k_zone = k_zone_data_buffers, 378 .max_k_zone = MAX_K_ZONE(k_zone_cfg) 379 }; 380 SECURITY_READ_ONLY_LATE(struct kalloc_heap) KHEAP_DATA_BUFFERS[1] = { 381 { 382 .kh_zones = &kalloc_zones_data_buffers, 383 .kh_name = "data.", 384 .kh_heap_id = KHEAP_ID_DATA_BUFFERS, 385 } 386 }; 387 388 389 /* 390 * Kext heap configuration 391 */ 392 static SECURITY_READ_ONLY_LATE(struct kheap_zones) kalloc_zones_kext = { 393 .cfg = k_zone_cfg, 394 .heap_id = KHEAP_ID_KEXT, 395 .k_zone = k_zone_kext, 396 .max_k_zone = MAX_K_ZONE(k_zone_cfg) 397 }; 398 SECURITY_READ_ONLY_LATE(struct kalloc_heap) KHEAP_KEXT[1] = { 399 { 400 .kh_zones = &kalloc_zones_kext, 401 .kh_name = "kext.", 402 .kh_heap_id = KHEAP_ID_KEXT, 403 } 404 }; 405 406 KALLOC_HEAP_DEFINE(KERN_OS_MALLOC, "kern_os_malloc", KHEAP_ID_KEXT); 407 408 /* 409 * Initialize kalloc heap: Create zones, generate direct lookup table and 410 * do a quick test on lookups 411 */ 412 __startup_func 413 static void 414 kalloc_zones_init(struct kheap_zones *zones) 415 { 416 struct kalloc_zone_cfg *cfg = zones->cfg; 417 zone_t *k_zone = zones->k_zone; 418 vm_size_t size; 419 420 /* 421 * Allocate a zone for each size we are going to handle. 422 */ 423 for (uint32_t i = 0; i < zones->max_k_zone && 424 (size = cfg[i].kzc_size) < kalloc_max; i++) { 425 zone_create_flags_t flags = ZC_KASAN_NOREDZONE | 426 ZC_KASAN_NOQUARANTINE | ZC_KALLOC_HEAP; 427 if (cfg[i].kzc_caching) { 428 flags |= ZC_CACHING; 429 } 430 431 k_zone[i] = zone_create_ext(cfg[i].kzc_name, size, flags, 432 ZONE_ID_ANY, ^(zone_t z){ 433 z->kalloc_heap = zones->heap_id; 434 }); 435 /* 436 * Set the updated elem size back to the config 437 */ 438 cfg[i].kzc_size = k_zone[i]->z_elem_size; 439 } 440 441 /* 442 * Count all the "raw" views for zones in the heap. 443 */ 444 zone_view_count += zones->max_k_zone; 445 446 /* 447 * Build the Direct LookUp Table for small allocations 448 * As k_zone_cfg is shared between the heaps the 449 * Direct LookUp Table is also shared and doesn't need to 450 * be rebuilt per heap. 451 */ 452 size = 0; 453 for (int i = 0; i <= KALLOC_DLUT_SIZE; i++, size += KALLOC_MINALIGN) { 454 uint8_t zindex = 0; 455 456 while ((vm_size_t)(cfg[zindex].kzc_size) < size) { 457 zindex++; 458 } 459 460 if (i == KALLOC_DLUT_SIZE) { 461 zones->k_zindex_start = zindex; 462 break; 463 } 464 zones->dlut[i] = zindex; 465 } 466 467 #ifdef KALLOC_DEBUG 468 printf("kalloc_init: k_zindex_start %d\n", zones->k_zindex_start); 469 470 /* 471 * Do a quick synthesis to see how well/badly we can 472 * find-a-zone for a given size. 473 * Useful when debugging/tweaking the array of zone sizes. 474 * Cache misses probably more critical than compare-branches! 475 */ 476 for (uint32_t i = 0; i < zones->max_k_zone; i++) { 477 vm_size_t testsize = (vm_size_t)(cfg[i].kzc_size - 1); 478 int compare = 0; 479 uint8_t zindex; 480 481 if (testsize < MAX_SIZE_ZDLUT) { 482 compare += 1; /* 'if' (T) */ 483 484 long dindex = INDEX_ZDLUT(testsize); 485 zindex = (int)zones->dlut[dindex]; 486 } else if (testsize < kalloc_max_prerounded) { 487 compare += 2; /* 'if' (F), 'if' (T) */ 488 489 zindex = zones->k_zindex_start; 490 while ((vm_size_t)(cfg[zindex].kzc_size) < testsize) { 491 zindex++; 492 compare++; /* 'while' (T) */ 493 } 494 compare++; /* 'while' (F) */ 495 } else { 496 break; /* not zone-backed */ 497 } 498 zone_t z = k_zone[zindex]; 499 printf("kalloc_init: req size %4lu: %8s.%16s took %d compare%s\n", 500 (unsigned long)testsize, kalloc_heap_names[zones->heap_id], 501 z->z_name, compare, compare == 1 ? "" : "s"); 502 } 503 #endif 504 } 505 506 /* 507 * Initialize the memory allocator. This should be called only 508 * once on a system wide basis (i.e. first processor to get here 509 * does the initialization). 510 * 511 * This initializes all of the zones. 512 */ 513 514 __startup_func 515 static void 516 kalloc_init(void) 517 { 518 kern_return_t retval; 519 vm_offset_t min; 520 vm_size_t kalloc_map_size; 521 vm_map_kernel_flags_t vmk_flags; 522 523 /* 524 * Scale the kalloc_map_size to physical memory size: stay below 525 * 1/8th the total zone map size, or 128 MB (for a 32-bit kernel). 526 */ 527 kalloc_map_size = (vm_size_t)(sane_size >> 5); 528 #if !__LP64__ 529 if (kalloc_map_size > KALLOC_MAP_SIZE_MAX) { 530 kalloc_map_size = KALLOC_MAP_SIZE_MAX; 531 } 532 #endif /* !__LP64__ */ 533 if (kalloc_map_size < KALLOC_MAP_SIZE_MIN) { 534 kalloc_map_size = KALLOC_MAP_SIZE_MIN; 535 } 536 537 vmk_flags = VM_MAP_KERNEL_FLAGS_NONE; 538 vmk_flags.vmkf_permanent = TRUE; 539 540 retval = kmem_suballoc(kernel_map, &min, kalloc_map_size, 541 FALSE, VM_FLAGS_ANYWHERE, vmk_flags, 542 VM_KERN_MEMORY_KALLOC, &kalloc_map); 543 544 if (retval != KERN_SUCCESS) { 545 panic("kalloc_init: kmem_suballoc failed"); 546 } 547 548 kalloc_map_min = min; 549 kalloc_map_max = min + kalloc_map_size - 1; 550 551 struct kheap_zones *khz_default = &kalloc_zones_default; 552 kalloc_max = (khz_default->cfg[khz_default->max_k_zone - 1].kzc_size << 1); 553 if (kalloc_max < KiB(16)) { 554 kalloc_max = KiB(16); 555 } 556 assert(kalloc_max <= KiB(64)); /* assumption made in size arrays */ 557 558 kalloc_max_prerounded = kalloc_max / 2 + 1; 559 /* allocations larger than 16 times kalloc_max go directly to kernel map */ 560 kalloc_kernmap_size = (kalloc_max * 16) + 1; 561 kalloc_largest_allocated = kalloc_kernmap_size; 562 563 /* Initialize kalloc default heap */ 564 kalloc_zones_init(&kalloc_zones_default); 565 566 /* Initialize kalloc data buffers heap */ 567 if (ZSECURITY_OPTIONS_SUBMAP_USER_DATA & zsecurity_options) { 568 kalloc_zones_init(&kalloc_zones_data_buffers); 569 } else { 570 *KHEAP_DATA_BUFFERS = *KHEAP_DEFAULT; 571 } 572 573 /* Initialize kalloc kext heap */ 574 if (ZSECURITY_OPTIONS_SEQUESTER_KEXT_KALLOC & zsecurity_options) { 575 kalloc_zones_init(&kalloc_zones_kext); 576 } else { 577 *KHEAP_KEXT = *KHEAP_DEFAULT; 578 } 579 } 580 STARTUP(ZALLOC, STARTUP_RANK_THIRD, kalloc_init); 581 582 583 #pragma mark accessors 584 585 static void 586 KALLOC_ZINFO_SALLOC(vm_size_t bytes) 587 { 588 thread_t thr = current_thread(); 589 ledger_debit_thread(thr, thr->t_ledger, task_ledgers.tkm_shared, bytes); 590 } 591 592 static void 593 KALLOC_ZINFO_SFREE(vm_size_t bytes) 594 { 595 thread_t thr = current_thread(); 596 ledger_credit_thread(thr, thr->t_ledger, task_ledgers.tkm_shared, bytes); 597 } 598 599 static inline vm_map_t 600 kalloc_map_for_addr(vm_address_t addr) 601 { 602 if (addr >= kalloc_map_min && addr < kalloc_map_max) { 603 return kalloc_map; 604 } 605 return kernel_map; 606 } 607 608 static inline vm_map_t 609 kalloc_map_for_size(vm_size_t size) 610 { 611 if (size < kalloc_kernmap_size) { 612 return kalloc_map; 613 } 614 return kernel_map; 615 } 616 617 zone_t 618 kalloc_heap_zone_for_size(kalloc_heap_t kheap, vm_size_t size) 619 { 620 struct kheap_zones *khz = kheap->kh_zones; 621 622 if (size < MAX_SIZE_ZDLUT) { 623 uint32_t zindex = khz->dlut[INDEX_ZDLUT(size)]; 624 return khz->k_zone[zindex]; 625 } 626 627 if (size < kalloc_max_prerounded) { 628 uint32_t zindex = khz->k_zindex_start; 629 while (khz->cfg[zindex].kzc_size < size) { 630 zindex++; 631 } 632 assert(zindex < khz->max_k_zone); 633 return khz->k_zone[zindex]; 634 } 635 636 return ZONE_NULL; 637 } 638 639 static vm_size_t 640 vm_map_lookup_kalloc_entry_locked(vm_map_t map, void *addr) 641 { 642 vm_map_entry_t vm_entry = NULL; 643 644 if (!vm_map_lookup_entry(map, (vm_map_offset_t)addr, &vm_entry)) { 645 panic("address %p not allocated via kalloc, map %p", 646 addr, map); 647 } 648 if (vm_entry->vme_start != (vm_map_offset_t)addr) { 649 panic("address %p inside vm entry %p [%p:%p), map %p", 650 addr, vm_entry, (void *)vm_entry->vme_start, 651 (void *)vm_entry->vme_end, map); 652 } 653 if (!vm_entry->vme_atomic) { 654 panic("address %p not managed by kalloc (entry %p, map %p)", 655 addr, vm_entry, map); 656 } 657 return vm_entry->vme_end - vm_entry->vme_start; 658 } 659 660 #if KASAN_KALLOC 661 /* 662 * KASAN kalloc stashes the original user-requested size away in the poisoned 663 * area. Return that directly. 664 */ 665 vm_size_t 666 kalloc_size(void *addr) 667 { 668 (void)vm_map_lookup_kalloc_entry_locked; /* silence warning */ 669 return kasan_user_size((vm_offset_t)addr); 670 } 671 #else 672 vm_size_t 673 kalloc_size(void *addr) 674 { 675 vm_map_t map; 676 vm_size_t size; 677 678 size = zone_element_size(addr, NULL); 679 if (size) { 680 return size; 681 } 682 683 map = kalloc_map_for_addr((vm_offset_t)addr); 684 vm_map_lock_read(map); 685 size = vm_map_lookup_kalloc_entry_locked(map, addr); 686 vm_map_unlock_read(map); 687 return size; 688 } 689 #endif 690 691 vm_size_t 692 kalloc_bucket_size(vm_size_t size) 693 { 694 zone_t z = kalloc_heap_zone_for_size(KHEAP_DEFAULT, size); 695 vm_map_t map = kalloc_map_for_size(size); 696 697 if (z) { 698 return zone_elem_size(z); 699 } 700 return vm_map_round_page(size, VM_MAP_PAGE_MASK(map)); 701 } 702 703 #pragma mark kalloc 704 705 void 706 kheap_temp_leak_panic(thread_t self) 707 { 708 #if DEBUG || DEVELOPMENT 709 if (__improbable(kheap_temp_debug)) { 710 struct kheap_temp_header *hdr = qe_dequeue_head(&self->t_temp_alloc_list, 711 struct kheap_temp_header, kht_hdr_link); 712 713 panic_plain("KHEAP_TEMP leak on thread %p (%d), allocated at:\n" 714 " %#016lx\n" " %#016lx\n" " %#016lx\n" " %#016lx\n" 715 " %#016lx\n" " %#016lx\n" " %#016lx\n" " %#016lx\n" 716 " %#016lx\n" " %#016lx\n" " %#016lx\n" " %#016lx\n" 717 " %#016lx\n" " %#016lx\n", 718 self, self->t_temp_alloc_count, 719 hdr->kht_hdr_pcs[0], hdr->kht_hdr_pcs[1], 720 hdr->kht_hdr_pcs[2], hdr->kht_hdr_pcs[3], 721 hdr->kht_hdr_pcs[4], hdr->kht_hdr_pcs[5], 722 hdr->kht_hdr_pcs[6], hdr->kht_hdr_pcs[7], 723 hdr->kht_hdr_pcs[8], hdr->kht_hdr_pcs[9], 724 hdr->kht_hdr_pcs[10], hdr->kht_hdr_pcs[11], 725 hdr->kht_hdr_pcs[12], hdr->kht_hdr_pcs[13]); 726 } 727 panic("KHEAP_TEMP leak on thread %p (%d) " 728 "(boot with kheap_temp_debug=1 to debug)", 729 self, self->t_temp_alloc_count); 730 #else /* !DEBUG && !DEVELOPMENT */ 731 panic("KHEAP_TEMP leak on thread %p (%d)", 732 self, self->t_temp_alloc_count); 733 #endif /* !DEBUG && !DEVELOPMENT */ 734 } 735 736 __abortlike 737 static void 738 kheap_temp_overuse_panic(thread_t self) 739 { 740 panic("too many KHEAP_TEMP allocations in flight: %d", 741 self->t_temp_alloc_count); 742 } 743 744 __attribute__((noinline)) 745 static struct kalloc_result 746 kalloc_large( 747 kalloc_heap_t kheap, 748 vm_size_t req_size, 749 vm_size_t size, 750 zalloc_flags_t flags, 751 vm_allocation_site_t *site) 752 { 753 int kma_flags = KMA_ATOMIC; 754 vm_tag_t tag; 755 vm_map_t alloc_map; 756 vm_offset_t addr; 757 758 if (flags & Z_NOFAIL) { 759 panic("trying to kalloc(Z_NOFAIL) with a large size (%zd)", 760 (size_t)size); 761 } 762 /* kmem_alloc could block so we return if noblock */ 763 if (flags & Z_NOWAIT) { 764 return (struct kalloc_result){ }; 765 } 766 767 #ifndef __x86_64__ 768 /* 769 * (73465472) on Intel we didn't use to pass this flag, 770 * which in turned allowed kalloc_large() memory to be shared 771 * with user directly. 772 * 773 * We're bound by this unfortunate ABI. 774 */ 775 kma_flags |= KMA_KOBJECT; 776 #endif 777 if (flags & Z_NOPAGEWAIT) { 778 kma_flags |= KMA_NOPAGEWAIT; 779 } 780 if (flags & Z_ZERO) { 781 kma_flags |= KMA_ZERO; 782 } 783 784 #if KASAN_KALLOC 785 /* large allocation - use guard pages instead of small redzones */ 786 size = round_page(req_size + 2 * PAGE_SIZE); 787 assert(size >= MAX_SIZE_ZDLUT && size >= kalloc_max_prerounded); 788 #else 789 size = round_page(size); 790 #endif 791 792 alloc_map = kalloc_map_for_size(size); 793 794 tag = zalloc_flags_get_tag(flags); 795 if (tag == VM_KERN_MEMORY_NONE) { 796 if (site) { 797 tag = vm_tag_alloc(site); 798 } else { 799 tag = VM_KERN_MEMORY_KALLOC; 800 } 801 } 802 803 if (kmem_alloc_flags(alloc_map, &addr, size, tag, kma_flags) != KERN_SUCCESS) { 804 if (alloc_map != kernel_map) { 805 if (kalloc_fallback_count++ == 0) { 806 printf("%s: falling back to kernel_map\n", __func__); 807 } 808 if (kmem_alloc_flags(kernel_map, &addr, size, tag, kma_flags) != KERN_SUCCESS) { 809 addr = 0; 810 } 811 } else { 812 addr = 0; 813 } 814 } 815 816 if (addr != 0) { 817 kalloc_spin_lock(); 818 /* 819 * Thread-safe version of the workaround for 4740071 820 * (a double FREE()) 821 */ 822 if (size > kalloc_largest_allocated) { 823 kalloc_largest_allocated = size; 824 } 825 826 kalloc_large_inuse++; 827 assert(kalloc_large_total + size >= kalloc_large_total); /* no wrap around */ 828 kalloc_large_total += size; 829 kalloc_large_sum += size; 830 831 if (kalloc_large_total > kalloc_large_max) { 832 kalloc_large_max = kalloc_large_total; 833 } 834 835 kalloc_unlock(); 836 837 KALLOC_ZINFO_SALLOC(size); 838 } 839 #if KASAN_KALLOC 840 /* fixup the return address to skip the redzone */ 841 addr = kasan_alloc(addr, size, req_size, PAGE_SIZE); 842 /* 843 * Initialize buffer with unique pattern only if memory 844 * wasn't expected to be zeroed. 845 */ 846 if (!(flags & Z_ZERO)) { 847 kasan_leak_init(addr, req_size); 848 } 849 #else 850 req_size = size; 851 #endif 852 853 if (addr && kheap == KHEAP_TEMP) { 854 thread_t self = current_thread(); 855 856 if (self->t_temp_alloc_count++ > UINT16_MAX) { 857 kheap_temp_overuse_panic(self); 858 } 859 #if DEBUG || DEVELOPMENT 860 if (__improbable(kheap_temp_debug)) { 861 struct kheap_temp_header *hdr = (void *)addr; 862 enqueue_head(&self->t_temp_alloc_list, 863 &hdr->kht_hdr_link); 864 backtrace(hdr->kht_hdr_pcs, KHT_BT_COUNT, NULL); 865 req_size -= sizeof(struct kheap_temp_header); 866 addr += sizeof(struct kheap_temp_header); 867 } 868 #endif /* DEBUG || DEVELOPMENT */ 869 } 870 871 DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, req_size, void*, addr); 872 return (struct kalloc_result){ .addr = (void *)addr, .size = req_size }; 873 } 874 875 struct kalloc_result 876 kalloc_ext( 877 kalloc_heap_t kheap, 878 vm_size_t req_size, 879 zalloc_flags_t flags, 880 vm_allocation_site_t *site) 881 { 882 vm_size_t size; 883 void *addr; 884 zone_t z; 885 886 #if DEBUG || DEVELOPMENT 887 if (__improbable(kheap_temp_debug)) { 888 if (kheap == KHEAP_TEMP) { 889 req_size += sizeof(struct kheap_temp_header); 890 } 891 } 892 #endif /* DEBUG || DEVELOPMENT */ 893 894 /* 895 * Kasan for kalloc heaps will put the redzones *inside* 896 * the allocation, and hence augment its size. 897 * 898 * kalloc heaps do not use zone_t::z_kasan_redzone. 899 */ 900 #if KASAN_KALLOC 901 size = kasan_alloc_resize(req_size); 902 #else 903 size = req_size; 904 #endif 905 z = kalloc_heap_zone_for_size(kheap, size); 906 if (__improbable(z == ZONE_NULL)) { 907 return kalloc_large(kheap, req_size, size, flags, site); 908 } 909 910 #ifdef KALLOC_DEBUG 911 if (size > zone_elem_size(z)) { 912 panic("%s: z %p (%s%s) but requested size %lu", __func__, z, 913 kalloc_heap_names[kheap->kh_zones->heap_id], z->z_name, 914 (unsigned long)size); 915 } 916 #endif 917 assert(size <= zone_elem_size(z)); 918 919 #if VM_MAX_TAG_ZONES 920 if (z->tags) { 921 vm_tag_t tag = zalloc_flags_get_tag(flags); 922 if (tag == VM_KERN_MEMORY_NONE && site) { 923 tag = vm_tag_alloc(site); 924 } 925 if (tag != VM_KERN_MEMORY_NONE) { 926 tag = vm_tag_will_update_zone(tag, z->tag_zone_index, 927 flags & (Z_WAITOK | Z_NOWAIT | Z_NOPAGEWAIT)); 928 } 929 flags |= Z_VM_TAG(tag); 930 } 931 #endif 932 addr = zalloc_ext(z, kheap->kh_stats ?: z->z_stats, flags); 933 934 #if KASAN_KALLOC 935 addr = (void *)kasan_alloc((vm_offset_t)addr, zone_elem_size(z), 936 req_size, KASAN_GUARD_SIZE); 937 #else 938 req_size = zone_elem_size(z); 939 #endif 940 941 if (addr && kheap == KHEAP_TEMP) { 942 thread_t self = current_thread(); 943 944 if (self->t_temp_alloc_count++ > UINT16_MAX) { 945 kheap_temp_overuse_panic(self); 946 } 947 #if DEBUG || DEVELOPMENT 948 if (__improbable(kheap_temp_debug)) { 949 struct kheap_temp_header *hdr = (void *)addr; 950 enqueue_head(&self->t_temp_alloc_list, 951 &hdr->kht_hdr_link); 952 backtrace(hdr->kht_hdr_pcs, KHT_BT_COUNT, NULL); 953 req_size -= sizeof(struct kheap_temp_header); 954 addr += sizeof(struct kheap_temp_header); 955 } 956 #endif /* DEBUG || DEVELOPMENT */ 957 } 958 959 DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, req_size, void*, addr); 960 return (struct kalloc_result){ .addr = addr, .size = req_size }; 961 } 962 963 void * 964 kalloc_external(vm_size_t size); 965 void * 966 kalloc_external(vm_size_t size) 967 { 968 return kheap_alloc_tag_bt(KHEAP_KEXT, size, Z_WAITOK, VM_KERN_MEMORY_KALLOC); 969 } 970 971 972 #pragma mark kfree 973 974 __attribute__((noinline)) 975 static void 976 kfree_large(vm_offset_t addr, vm_size_t size) 977 { 978 vm_map_t map = kalloc_map_for_addr(addr); 979 kern_return_t ret; 980 vm_offset_t end; 981 982 if (addr < VM_MIN_KERNEL_AND_KEXT_ADDRESS || 983 os_add_overflow(addr, size, &end) || 984 end > VM_MAX_KERNEL_ADDRESS) { 985 panic("kfree: address range (%p, %ld) doesn't belong to the kernel", 986 (void *)addr, (uintptr_t)size); 987 } 988 989 if (size == 0) { 990 vm_map_lock(map); 991 size = vm_map_lookup_kalloc_entry_locked(map, (void *)addr); 992 ret = vm_map_remove_locked(map, 993 vm_map_trunc_page(addr, VM_MAP_PAGE_MASK(map)), 994 vm_map_round_page(addr + size, VM_MAP_PAGE_MASK(map)), 995 VM_MAP_REMOVE_KUNWIRE); 996 if (ret != KERN_SUCCESS) { 997 panic("kfree: vm_map_remove_locked() failed for " 998 "addr: %p, map: %p ret: %d", (void *)addr, map, ret); 999 } 1000 vm_map_unlock(map); 1001 } else { 1002 size = round_page(size); 1003 1004 if (size > kalloc_largest_allocated) { 1005 panic("kfree: size %lu > kalloc_largest_allocated %lu", 1006 (uintptr_t)size, (uintptr_t)kalloc_largest_allocated); 1007 } 1008 kmem_free(map, addr, size); 1009 } 1010 1011 kalloc_spin_lock(); 1012 1013 assert(kalloc_large_total >= size); 1014 kalloc_large_total -= size; 1015 kalloc_large_inuse--; 1016 1017 kalloc_unlock(); 1018 1019 #if !KASAN_KALLOC 1020 DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, size, void*, addr); 1021 #endif 1022 1023 KALLOC_ZINFO_SFREE(size); 1024 return; 1025 } 1026 1027 __abortlike 1028 static void 1029 kfree_heap_confusion_panic(kalloc_heap_t kheap, void *data, size_t size, zone_t z) 1030 { 1031 if (z->kalloc_heap == KHEAP_ID_NONE) { 1032 panic("kfree: addr %p, size %zd found in regular zone '%s%s'", 1033 data, size, zone_heap_name(z), z->z_name); 1034 } else { 1035 panic("kfree: addr %p, size %zd found in heap %s* instead of %s*", 1036 data, size, zone_heap_name(z), 1037 kalloc_heap_names[kheap->kh_heap_id]); 1038 } 1039 } 1040 1041 __abortlike 1042 static void 1043 kfree_size_confusion_panic(zone_t z, void *data, size_t size, size_t zsize) 1044 { 1045 if (z) { 1046 panic("kfree: addr %p, size %zd found in zone '%s%s' " 1047 "with elem_size %zd", 1048 data, size, zone_heap_name(z), z->z_name, zsize); 1049 } else { 1050 panic("kfree: addr %p, size %zd not found in any zone", 1051 data, size); 1052 } 1053 } 1054 1055 __abortlike 1056 static void 1057 kfree_size_invalid_panic(void *data, size_t size) 1058 { 1059 panic("kfree: addr %p trying to free with nonsensical size %zd", 1060 data, size); 1061 } 1062 1063 __abortlike 1064 static void 1065 krealloc_size_invalid_panic(void *data, size_t size) 1066 { 1067 panic("krealloc: addr %p trying to free with nonsensical size %zd", 1068 data, size); 1069 } 1070 1071 __abortlike 1072 static void 1073 kfree_temp_imbalance_panic(void *data, size_t size) 1074 { 1075 panic("kfree: KHEAP_TEMP allocation imbalance freeing addr %p, size %zd", 1076 data, size); 1077 } 1078 1079 /* used to implement kheap_free_addr() */ 1080 #define KFREE_UNKNOWN_SIZE ((vm_size_t)~0) 1081 #define KFREE_ABSURD_SIZE \ 1082 ((VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_AND_KEXT_ADDRESS) / 2) 1083 1084 static void 1085 kfree_ext(kalloc_heap_t kheap, void *data, vm_size_t size) 1086 { 1087 zone_stats_t zs = NULL; 1088 zone_t z; 1089 vm_size_t zsize; 1090 1091 if (__improbable(data == NULL)) { 1092 return; 1093 } 1094 1095 if (kheap == KHEAP_TEMP) { 1096 assert(size != KFREE_UNKNOWN_SIZE); 1097 if (current_thread()->t_temp_alloc_count-- == 0) { 1098 kfree_temp_imbalance_panic(data, size); 1099 } 1100 #if DEBUG || DEVELOPMENT 1101 if (__improbable(kheap_temp_debug)) { 1102 size += sizeof(struct kheap_temp_header); 1103 data -= sizeof(struct kheap_temp_header); 1104 remqueue(&((struct kheap_temp_header *)data)->kht_hdr_link); 1105 } 1106 #endif /* DEBUG || DEVELOPMENT */ 1107 } 1108 1109 #if KASAN_KALLOC 1110 /* 1111 * Resize back to the real allocation size and hand off to the KASan 1112 * quarantine. `data` may then point to a different allocation. 1113 */ 1114 vm_size_t user_size = size; 1115 if (size == KFREE_UNKNOWN_SIZE) { 1116 user_size = size = kalloc_size(data); 1117 } 1118 kasan_check_free((vm_address_t)data, size, KASAN_HEAP_KALLOC); 1119 data = (void *)kasan_dealloc((vm_address_t)data, &size); 1120 kasan_free(&data, &size, KASAN_HEAP_KALLOC, NULL, user_size, true); 1121 if (!data) { 1122 return; 1123 } 1124 #endif 1125 1126 if (size >= kalloc_max_prerounded && size != KFREE_UNKNOWN_SIZE) { 1127 return kfree_large((vm_offset_t)data, size); 1128 } 1129 1130 zsize = zone_element_size(data, &z); 1131 if (size == KFREE_UNKNOWN_SIZE) { 1132 if (zsize == 0) { 1133 return kfree_large((vm_offset_t)data, 0); 1134 } 1135 size = zsize; 1136 } else if (size > zsize) { 1137 kfree_size_confusion_panic(z, data, size, zsize); 1138 } 1139 1140 if (kheap != KHEAP_ANY) { 1141 if (kheap->kh_heap_id != z->kalloc_heap) { 1142 kfree_heap_confusion_panic(kheap, data, size, z); 1143 } 1144 zs = kheap->kh_stats; 1145 } else if (z->kalloc_heap != KHEAP_ID_DEFAULT && 1146 z->kalloc_heap != KHEAP_ID_KEXT) { 1147 kfree_heap_confusion_panic(kheap, data, size, z); 1148 } 1149 1150 #if !KASAN_KALLOC 1151 DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, zsize, void*, data); 1152 #endif 1153 zfree_ext(z, zs ?: z->z_stats, data); 1154 } 1155 1156 void 1157 (kfree)(void *addr, vm_size_t size) 1158 { 1159 if (size > KFREE_ABSURD_SIZE) { 1160 kfree_size_invalid_panic(addr, size); 1161 } 1162 kfree_ext(KHEAP_ANY, addr, size); 1163 } 1164 1165 void 1166 (kheap_free)(kalloc_heap_t kheap, void *addr, vm_size_t size) 1167 { 1168 if (size > KFREE_ABSURD_SIZE) { 1169 kfree_size_invalid_panic(addr, size); 1170 } 1171 kfree_ext(kheap, addr, size); 1172 } 1173 1174 void 1175 (kheap_free_addr)(kalloc_heap_t kheap, void *addr) 1176 { 1177 kfree_ext(kheap, addr, KFREE_UNKNOWN_SIZE); 1178 } 1179 1180 static struct kalloc_result 1181 _krealloc_ext( 1182 kalloc_heap_t kheap, 1183 void *addr, 1184 vm_size_t old_size, 1185 vm_size_t new_size, 1186 zalloc_flags_t flags, 1187 vm_allocation_site_t *site) 1188 { 1189 vm_size_t old_bucket_size, new_bucket_size, min_size; 1190 struct kalloc_result kr; 1191 1192 if (new_size == 0) { 1193 kfree_ext(kheap, addr, old_size); 1194 return (struct kalloc_result){ }; 1195 } 1196 1197 if (addr == NULL) { 1198 return kalloc_ext(kheap, new_size, flags, site); 1199 } 1200 1201 /* 1202 * Find out the size of the bucket in which the new sized allocation 1203 * would land. If it matches the bucket of the original allocation, 1204 * simply return the same address. 1205 */ 1206 new_bucket_size = kalloc_bucket_size(new_size); 1207 if (old_size == KFREE_UNKNOWN_SIZE) { 1208 old_size = old_bucket_size = kalloc_size(addr); 1209 } else { 1210 old_bucket_size = kalloc_bucket_size(old_size); 1211 } 1212 min_size = MIN(old_size, new_size); 1213 1214 if (old_bucket_size == new_bucket_size) { 1215 kr.addr = addr; 1216 #if KASAN_KALLOC 1217 kr.size = new_size; 1218 #else 1219 kr.size = new_bucket_size; 1220 #endif 1221 } else { 1222 kr = kalloc_ext(kheap, new_size, flags & ~Z_ZERO, site); 1223 if (kr.addr == NULL) { 1224 return kr; 1225 } 1226 1227 memcpy(kr.addr, addr, min_size); 1228 kfree_ext(kheap, addr, old_size); 1229 } 1230 if ((flags & Z_ZERO) && kr.size > min_size) { 1231 bzero(kr.addr + min_size, kr.size - min_size); 1232 } 1233 return kr; 1234 } 1235 1236 struct kalloc_result 1237 krealloc_ext( 1238 kalloc_heap_t kheap, 1239 void *addr, 1240 vm_size_t old_size, 1241 vm_size_t new_size, 1242 zalloc_flags_t flags, 1243 vm_allocation_site_t *site) 1244 { 1245 if (old_size > KFREE_ABSURD_SIZE) { 1246 krealloc_size_invalid_panic(addr, old_size); 1247 } 1248 return _krealloc_ext(kheap, addr, old_size, new_size, flags, site); 1249 } 1250 1251 struct kalloc_result 1252 kheap_realloc_addr( 1253 kalloc_heap_t kheap, 1254 void *addr, 1255 vm_size_t size, 1256 zalloc_flags_t flags, 1257 vm_allocation_site_t *site) 1258 { 1259 return _krealloc_ext(kheap, addr, KFREE_UNKNOWN_SIZE, size, flags, site); 1260 } 1261 1262 __startup_func 1263 void 1264 kheap_startup_init(kalloc_heap_t kheap) 1265 { 1266 struct kheap_zones *zones; 1267 1268 switch (kheap->kh_heap_id) { 1269 case KHEAP_ID_DEFAULT: 1270 zones = KHEAP_DEFAULT->kh_zones; 1271 break; 1272 case KHEAP_ID_DATA_BUFFERS: 1273 zones = KHEAP_DATA_BUFFERS->kh_zones; 1274 break; 1275 case KHEAP_ID_KEXT: 1276 zones = KHEAP_KEXT->kh_zones; 1277 break; 1278 default: 1279 panic("kalloc_heap_startup_init: invalid KHEAP_ID: %d", 1280 kheap->kh_heap_id); 1281 } 1282 1283 kheap->kh_heap_id = zones->heap_id; 1284 kheap->kh_zones = zones; 1285 kheap->kh_stats = zalloc_percpu_permanent_type(struct zone_stats); 1286 kheap->kh_next = zones->views; 1287 zones->views = kheap; 1288 1289 zone_view_count += 1; 1290 } 1291 1292 #pragma mark OSMalloc 1293 /* 1294 * This is a deprecated interface, here only for legacy reasons. 1295 * There is no internal variant of any of these symbols on purpose. 1296 */ 1297 #define OSMallocDeprecated 1298 #include <libkern/OSMalloc.h> 1299 1300 static KALLOC_HEAP_DEFINE(OSMALLOC, "osmalloc", KHEAP_ID_KEXT); 1301 static queue_head_t OSMalloc_tag_list = QUEUE_HEAD_INITIALIZER(OSMalloc_tag_list); 1302 static LCK_GRP_DECLARE(OSMalloc_tag_lck_grp, "OSMalloc_tag"); 1303 static LCK_SPIN_DECLARE(OSMalloc_tag_lock, &OSMalloc_tag_lck_grp); 1304 1305 #define OSMalloc_tag_spin_lock() lck_spin_lock(&OSMalloc_tag_lock) 1306 #define OSMalloc_tag_unlock() lck_spin_unlock(&OSMalloc_tag_lock) 1307 1308 extern typeof(OSMalloc_Tagalloc) OSMalloc_Tagalloc_external; 1309 OSMallocTag 1310 OSMalloc_Tagalloc_external(const char *str, uint32_t flags) 1311 { 1312 OSMallocTag OSMTag; 1313 1314 OSMTag = kheap_alloc(OSMALLOC, sizeof(*OSMTag), Z_WAITOK | Z_ZERO); 1315 1316 if (flags & OSMT_PAGEABLE) { 1317 OSMTag->OSMT_attr = OSMT_ATTR_PAGEABLE; 1318 } 1319 1320 OSMTag->OSMT_refcnt = 1; 1321 1322 strlcpy(OSMTag->OSMT_name, str, OSMT_MAX_NAME); 1323 1324 OSMalloc_tag_spin_lock(); 1325 enqueue_tail(&OSMalloc_tag_list, (queue_entry_t)OSMTag); 1326 OSMalloc_tag_unlock(); 1327 OSMTag->OSMT_state = OSMT_VALID; 1328 return OSMTag; 1329 } 1330 1331 static void 1332 OSMalloc_Tagref(OSMallocTag tag) 1333 { 1334 if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) { 1335 panic("OSMalloc_Tagref():'%s' has bad state 0x%08X\n", 1336 tag->OSMT_name, tag->OSMT_state); 1337 } 1338 1339 os_atomic_inc(&tag->OSMT_refcnt, relaxed); 1340 } 1341 1342 static void 1343 OSMalloc_Tagrele(OSMallocTag tag) 1344 { 1345 if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) { 1346 panic("OSMalloc_Tagref():'%s' has bad state 0x%08X\n", 1347 tag->OSMT_name, tag->OSMT_state); 1348 } 1349 1350 if (os_atomic_dec(&tag->OSMT_refcnt, relaxed) != 0) { 1351 return; 1352 } 1353 1354 if (os_atomic_cmpxchg(&tag->OSMT_state, 1355 OSMT_VALID | OSMT_RELEASED, OSMT_VALID | OSMT_RELEASED, acq_rel)) { 1356 OSMalloc_tag_spin_lock(); 1357 (void)remque((queue_entry_t)tag); 1358 OSMalloc_tag_unlock(); 1359 kheap_free(OSMALLOC, tag, sizeof(*tag)); 1360 } else { 1361 panic("OSMalloc_Tagrele():'%s' has refcnt 0\n", tag->OSMT_name); 1362 } 1363 } 1364 1365 extern typeof(OSMalloc_Tagfree) OSMalloc_Tagfree_external; 1366 void 1367 OSMalloc_Tagfree_external(OSMallocTag tag) 1368 { 1369 if (!os_atomic_cmpxchg(&tag->OSMT_state, 1370 OSMT_VALID, OSMT_VALID | OSMT_RELEASED, acq_rel)) { 1371 panic("OSMalloc_Tagfree():'%s' has bad state 0x%08X \n", 1372 tag->OSMT_name, tag->OSMT_state); 1373 } 1374 1375 if (os_atomic_dec(&tag->OSMT_refcnt, relaxed) == 0) { 1376 OSMalloc_tag_spin_lock(); 1377 (void)remque((queue_entry_t)tag); 1378 OSMalloc_tag_unlock(); 1379 kheap_free(OSMALLOC, tag, sizeof(*tag)); 1380 } 1381 } 1382 1383 extern typeof(OSMalloc) OSMalloc_external; 1384 void * 1385 OSMalloc_external( 1386 uint32_t size, OSMallocTag tag) 1387 { 1388 void *addr = NULL; 1389 kern_return_t kr; 1390 1391 OSMalloc_Tagref(tag); 1392 if ((tag->OSMT_attr & OSMT_PAGEABLE) && (size & ~PAGE_MASK)) { 1393 if ((kr = kmem_alloc_pageable_external(kernel_map, 1394 (vm_offset_t *)&addr, size)) != KERN_SUCCESS) { 1395 addr = NULL; 1396 } 1397 } else { 1398 addr = kheap_alloc_tag_bt(OSMALLOC, size, 1399 Z_WAITOK, VM_KERN_MEMORY_KALLOC); 1400 } 1401 1402 if (!addr) { 1403 OSMalloc_Tagrele(tag); 1404 } 1405 1406 return addr; 1407 } 1408 1409 extern typeof(OSMalloc_nowait) OSMalloc_nowait_external; 1410 void * 1411 OSMalloc_nowait_external(uint32_t size, OSMallocTag tag) 1412 { 1413 void *addr = NULL; 1414 1415 if (tag->OSMT_attr & OSMT_PAGEABLE) { 1416 return NULL; 1417 } 1418 1419 OSMalloc_Tagref(tag); 1420 /* XXX: use non-blocking kalloc for now */ 1421 addr = kheap_alloc_tag_bt(OSMALLOC, (vm_size_t)size, 1422 Z_NOWAIT, VM_KERN_MEMORY_KALLOC); 1423 if (addr == NULL) { 1424 OSMalloc_Tagrele(tag); 1425 } 1426 1427 return addr; 1428 } 1429 1430 extern typeof(OSMalloc_noblock) OSMalloc_noblock_external; 1431 void * 1432 OSMalloc_noblock_external(uint32_t size, OSMallocTag tag) 1433 { 1434 void *addr = NULL; 1435 1436 if (tag->OSMT_attr & OSMT_PAGEABLE) { 1437 return NULL; 1438 } 1439 1440 OSMalloc_Tagref(tag); 1441 addr = kheap_alloc_tag_bt(OSMALLOC, (vm_size_t)size, 1442 Z_NOWAIT, VM_KERN_MEMORY_KALLOC); 1443 if (addr == NULL) { 1444 OSMalloc_Tagrele(tag); 1445 } 1446 1447 return addr; 1448 } 1449 1450 extern typeof(OSFree) OSFree_external; 1451 void 1452 OSFree_external(void *addr, uint32_t size, OSMallocTag tag) 1453 { 1454 if ((tag->OSMT_attr & OSMT_PAGEABLE) 1455 && (size & ~PAGE_MASK)) { 1456 kmem_free(kernel_map, (vm_offset_t)addr, size); 1457 } else { 1458 kheap_free(OSMALLOC, addr, size); 1459 } 1460 1461 OSMalloc_Tagrele(tag); 1462 } 1463 1464 #pragma mark kern_os_malloc 1465 1466 void * 1467 kern_os_malloc_external(size_t size); 1468 void * 1469 kern_os_malloc_external(size_t size) 1470 { 1471 if (size == 0) { 1472 return NULL; 1473 } 1474 1475 return kheap_alloc_tag_bt(KERN_OS_MALLOC, size, Z_WAITOK | Z_ZERO, 1476 VM_KERN_MEMORY_LIBKERN); 1477 } 1478 1479 void 1480 kern_os_free_external(void *addr); 1481 void 1482 kern_os_free_external(void *addr) 1483 { 1484 kheap_free_addr(KERN_OS_MALLOC, addr); 1485 } 1486 1487 void * 1488 kern_os_realloc_external(void *addr, size_t nsize); 1489 void * 1490 kern_os_realloc_external(void *addr, size_t nsize) 1491 { 1492 VM_ALLOC_SITE_STATIC(VM_TAG_BT, VM_KERN_MEMORY_LIBKERN); 1493 1494 return kheap_realloc_addr(KERN_OS_MALLOC, addr, nsize, 1495 Z_WAITOK | Z_ZERO, &site).addr; 1496 } 1497 1498 void 1499 kern_os_zfree(zone_t zone, void *addr, vm_size_t size) 1500 { 1501 if (zsecurity_options & ZSECURITY_OPTIONS_STRICT_IOKIT_FREE 1502 || zone_owns(zone, addr)) { 1503 zfree(zone, addr); 1504 } else { 1505 /* 1506 * Third party kexts might not know about the operator new 1507 * and be allocated from the KEXT heap 1508 */ 1509 printf("kern_os_zfree: kheap_free called for object from zone %s\n", 1510 zone->z_name); 1511 kheap_free(KHEAP_KEXT, addr, size); 1512 } 1513 } 1514 1515 void 1516 kern_os_kfree(void *addr, vm_size_t size) 1517 { 1518 if (zsecurity_options & ZSECURITY_OPTIONS_STRICT_IOKIT_FREE) { 1519 kheap_free(KHEAP_DEFAULT, addr, size); 1520 } else { 1521 /* 1522 * Third party kexts may not know about newly added operator 1523 * default new/delete. If they call new for any iokit object 1524 * it will end up coming from the KEXT heap. If these objects 1525 * are freed by calling release() or free(), the internal 1526 * version of operator delete is called and the kernel ends 1527 * up freeing the object to the DEFAULT heap. 1528 */ 1529 kheap_free(KHEAP_ANY, addr, size); 1530 } 1531 }