thread.c
1 /* 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_FREE_COPYRIGHT@ 30 */ 31 /* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56 /* 57 */ 58 /* 59 * File: kern/thread.c 60 * Author: Avadis Tevanian, Jr., Michael Wayne Young, David Golub 61 * Date: 1986 62 * 63 * Thread management primitives implementation. 64 */ 65 /* 66 * Copyright (c) 1993 The University of Utah and 67 * the Computer Systems Laboratory (CSL). All rights reserved. 68 * 69 * Permission to use, copy, modify and distribute this software and its 70 * documentation is hereby granted, provided that both the copyright 71 * notice and this permission notice appear in all copies of the 72 * software, derivative works or modified versions, and any portions 73 * thereof, and that both notices appear in supporting documentation. 74 * 75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS 76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF 77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 78 * 79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any 80 * improvements that they make and grant CSL redistribution rights. 81 * 82 */ 83 84 #include <mach/mach_types.h> 85 #include <mach/boolean.h> 86 #include <mach/policy.h> 87 #include <mach/thread_info.h> 88 #include <mach/thread_special_ports.h> 89 #include <mach/thread_status.h> 90 #include <mach/time_value.h> 91 #include <mach/vm_param.h> 92 93 #include <machine/thread.h> 94 #include <machine/pal_routines.h> 95 #include <machine/limits.h> 96 97 #include <kern/kern_types.h> 98 #include <kern/kalloc.h> 99 #include <kern/cpu_data.h> 100 #include <kern/extmod_statistics.h> 101 #include <kern/ipc_mig.h> 102 #include <kern/ipc_tt.h> 103 #include <kern/mach_param.h> 104 #include <kern/machine.h> 105 #include <kern/misc_protos.h> 106 #include <kern/processor.h> 107 #include <kern/queue.h> 108 #include <kern/sched.h> 109 #include <kern/sched_prim.h> 110 #include <kern/sync_lock.h> 111 #include <kern/syscall_subr.h> 112 #include <kern/task.h> 113 #include <kern/thread.h> 114 #include <kern/thread_group.h> 115 #include <kern/coalition.h> 116 #include <kern/host.h> 117 #include <kern/zalloc.h> 118 #include <kern/assert.h> 119 #include <kern/exc_resource.h> 120 #include <kern/exc_guard.h> 121 #include <kern/telemetry.h> 122 #include <kern/policy_internal.h> 123 #include <kern/turnstile.h> 124 #include <kern/sched_clutch.h> 125 126 #include <corpses/task_corpse.h> 127 #if KPC 128 #include <kern/kpc.h> 129 #endif 130 131 #if MONOTONIC 132 #include <kern/monotonic.h> 133 #include <machine/monotonic.h> 134 #endif /* MONOTONIC */ 135 136 #include <ipc/ipc_kmsg.h> 137 #include <ipc/ipc_port.h> 138 #include <bank/bank_types.h> 139 140 #include <vm/vm_kern.h> 141 #include <vm/vm_pageout.h> 142 143 #include <sys/kdebug.h> 144 #include <sys/bsdtask_info.h> 145 #include <mach/sdt.h> 146 #include <san/kasan.h> 147 #if CONFIG_KSANCOV 148 #include <san/ksancov.h> 149 #endif 150 151 #include <stdatomic.h> 152 153 #if defined(HAS_APPLE_PAC) 154 #include <ptrauth.h> 155 #include <arm64/proc_reg.h> 156 #endif /* defined(HAS_APPLE_PAC) */ 157 158 /* 159 * Exported interfaces 160 */ 161 #include <mach/task_server.h> 162 #include <mach/thread_act_server.h> 163 #include <mach/mach_host_server.h> 164 #include <mach/host_priv_server.h> 165 #include <mach/mach_voucher_server.h> 166 #include <kern/policy_internal.h> 167 168 #if CONFIG_MACF 169 #include <security/mac_mach_internal.h> 170 #endif 171 172 LCK_GRP_DECLARE(thread_lck_grp, "thread"); 173 174 ZONE_DECLARE(thread_zone, "threads", sizeof(struct thread), ZC_ZFREE_CLEARMEM); 175 176 ZONE_DECLARE(thread_qos_override_zone, "thread qos override", 177 sizeof(struct thread_qos_override), ZC_NOENCRYPT); 178 179 static struct mpsc_daemon_queue thread_stack_queue; 180 static struct mpsc_daemon_queue thread_terminate_queue; 181 static struct mpsc_daemon_queue thread_deallocate_queue; 182 static struct mpsc_daemon_queue thread_exception_queue; 183 184 decl_simple_lock_data(static, crashed_threads_lock); 185 static queue_head_t crashed_threads_queue; 186 187 struct thread_exception_elt { 188 struct mpsc_queue_chain link; 189 exception_type_t exception_type; 190 task_t exception_task; 191 thread_t exception_thread; 192 }; 193 194 static SECURITY_READ_ONLY_LATE(struct thread) thread_template = { 195 #if MACH_ASSERT 196 .thread_magic = THREAD_MAGIC, 197 #endif /* MACH_ASSERT */ 198 .wait_result = THREAD_WAITING, 199 .options = THREAD_ABORTSAFE, 200 .state = TH_WAIT | TH_UNINT, 201 .th_sched_bucket = TH_BUCKET_RUN, 202 .base_pri = BASEPRI_DEFAULT, 203 .realtime.deadline = UINT64_MAX, 204 .last_made_runnable_time = THREAD_NOT_RUNNABLE, 205 .last_basepri_change_time = THREAD_NOT_RUNNABLE, 206 #if defined(CONFIG_SCHED_TIMESHARE_CORE) 207 .pri_shift = INT8_MAX, 208 #endif 209 /* timers are initialized in thread_bootstrap */ 210 }; 211 212 static struct thread init_thread; 213 static void thread_deallocate_enqueue(thread_t thread); 214 static void thread_deallocate_complete(thread_t thread); 215 216 #ifdef MACH_BSD 217 extern void proc_exit(void *); 218 extern mach_exception_data_type_t proc_encode_exit_exception_code(void *); 219 extern uint64_t get_dispatchqueue_offset_from_proc(void *); 220 extern uint64_t get_return_to_kernel_offset_from_proc(void *p); 221 extern int proc_selfpid(void); 222 extern void proc_name(int, char*, int); 223 extern char * proc_name_address(void *p); 224 #endif /* MACH_BSD */ 225 226 extern int disable_exc_resource; 227 extern int audio_active; 228 extern int debug_task; 229 int thread_max = CONFIG_THREAD_MAX; /* Max number of threads */ 230 int task_threadmax = CONFIG_THREAD_MAX; 231 232 static uint64_t thread_unique_id = 100; 233 234 struct _thread_ledger_indices thread_ledgers = { .cpu_time = -1 }; 235 static ledger_template_t thread_ledger_template = NULL; 236 static void init_thread_ledgers(void); 237 238 #if CONFIG_JETSAM 239 void jetsam_on_ledger_cpulimit_exceeded(void); 240 #endif 241 242 extern int task_thread_soft_limit; 243 extern int exc_via_corpse_forking; 244 245 #if DEVELOPMENT || DEBUG 246 extern int exc_resource_threads_enabled; 247 #endif /* DEVELOPMENT || DEBUG */ 248 249 /* 250 * Level (in terms of percentage of the limit) at which the CPU usage monitor triggers telemetry. 251 * 252 * (ie when any thread's CPU consumption exceeds 70% of the limit, start taking user 253 * stacktraces, aka micro-stackshots) 254 */ 255 #define CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT 70 256 257 /* Percentage. Level at which we start gathering telemetry. */ 258 static TUNABLE(uint8_t, cpumon_ustackshots_trigger_pct, 259 "cpumon_ustackshots_trigger_pct", CPUMON_USTACKSHOTS_TRIGGER_DEFAULT_PCT); 260 void __attribute__((noinline)) SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void); 261 #if DEVELOPMENT || DEBUG 262 void __attribute__((noinline)) SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t, int); 263 #endif /* DEVELOPMENT || DEBUG */ 264 265 /* 266 * The smallest interval over which we support limiting CPU consumption is 1ms 267 */ 268 #define MINIMUM_CPULIMIT_INTERVAL_MS 1 269 270 os_refgrp_decl(static, thread_refgrp, "thread", NULL); 271 272 static inline void 273 init_thread_from_template(thread_t thread) 274 { 275 /* 276 * In general, struct thread isn't trivially-copyable, since it may 277 * contain pointers to thread-specific state. This may be enforced at 278 * compile time on architectures that store authed + diversified 279 * pointers in machine_thread. 280 * 281 * In this specific case, where we're initializing a new thread from a 282 * thread_template, we know all diversified pointers are NULL; these are 283 * safe to bitwise copy. 284 */ 285 #pragma clang diagnostic push 286 #pragma clang diagnostic ignored "-Wnontrivial-memaccess" 287 memcpy(thread, &thread_template, sizeof(*thread)); 288 #pragma clang diagnostic pop 289 } 290 291 thread_t 292 thread_bootstrap(void) 293 { 294 /* 295 * Fill in a template thread for fast initialization. 296 */ 297 timer_init(&thread_template.user_timer); 298 timer_init(&thread_template.system_timer); 299 timer_init(&thread_template.ptime); 300 timer_init(&thread_template.runnable_timer); 301 302 init_thread_from_template(&init_thread); 303 /* fiddle with init thread to skip asserts in set_sched_pri */ 304 init_thread.sched_pri = MAXPRI_KERNEL; 305 #if DEBUG || DEVELOPMENT 306 queue_init(&init_thread.t_temp_alloc_list); 307 #endif /* DEBUG || DEVELOPMENT */ 308 309 return &init_thread; 310 } 311 312 void 313 thread_machine_init_template(void) 314 { 315 machine_thread_template_init(&thread_template); 316 } 317 318 void 319 thread_init(void) 320 { 321 stack_init(); 322 323 thread_policy_init(); 324 325 /* 326 * Initialize any machine-dependent 327 * per-thread structures necessary. 328 */ 329 machine_thread_init(); 330 331 init_thread_ledgers(); 332 } 333 334 boolean_t 335 thread_is_active(thread_t thread) 336 { 337 return thread->active; 338 } 339 340 void 341 thread_corpse_continue(void) 342 { 343 thread_t thread = current_thread(); 344 345 thread_terminate_internal(thread, TH_TERMINATE_OPTION_NONE); 346 347 /* 348 * Handle the thread termination directly 349 * here instead of returning to userspace. 350 */ 351 assert(thread->active == FALSE); 352 thread_ast_clear(thread, AST_APC); 353 thread_apc_ast(thread); 354 355 panic("thread_corpse_continue"); 356 /*NOTREACHED*/ 357 } 358 359 __dead2 360 static void 361 thread_terminate_continue(void) 362 { 363 panic("thread_terminate_continue"); 364 /*NOTREACHED*/ 365 } 366 367 /* 368 * thread_terminate_self: 369 */ 370 void 371 thread_terminate_self(void) 372 { 373 thread_t thread = current_thread(); 374 task_t task; 375 int threadcnt; 376 377 if (thread->t_temp_alloc_count) { 378 kheap_temp_leak_panic(thread); 379 } 380 381 pal_thread_terminate_self(thread); 382 383 DTRACE_PROC(lwp__exit); 384 385 thread_mtx_lock(thread); 386 387 ipc_thread_disable(thread); 388 389 thread_mtx_unlock(thread); 390 391 thread_sched_call(thread, NULL); 392 393 spl_t s = splsched(); 394 thread_lock(thread); 395 396 thread_depress_abort_locked(thread); 397 398 thread_unlock(thread); 399 splx(s); 400 401 #if CONFIG_TASKWATCH 402 thead_remove_taskwatch(thread); 403 #endif /* CONFIG_TASKWATCH */ 404 405 work_interval_thread_terminate(thread); 406 407 thread_mtx_lock(thread); 408 409 thread_policy_reset(thread); 410 411 thread_mtx_unlock(thread); 412 413 assert(thread->th_work_interval == NULL); 414 415 bank_swap_thread_bank_ledger(thread, NULL); 416 417 if (kdebug_enable && bsd_hasthreadname(thread->uthread)) { 418 char threadname[MAXTHREADNAMESIZE]; 419 bsd_getthreadname(thread->uthread, threadname); 420 kernel_debug_string_simple(TRACE_STRING_THREADNAME_PREV, threadname); 421 } 422 423 task = thread->task; 424 uthread_cleanup(task, thread->uthread, task->bsd_info); 425 426 if (kdebug_enable && task->bsd_info && !task_is_exec_copy(task)) { 427 /* trace out pid before we sign off */ 428 long dbg_arg1 = 0; 429 long dbg_arg2 = 0; 430 431 kdbg_trace_data(thread->task->bsd_info, &dbg_arg1, &dbg_arg2); 432 #if MONOTONIC 433 if (kdebug_debugid_enabled(DBG_MT_INSTRS_CYCLES_THR_EXIT)) { 434 uint64_t counts[MT_CORE_NFIXED]; 435 uint64_t thread_user_time; 436 uint64_t thread_system_time; 437 thread_user_time = timer_grab(&thread->user_timer); 438 thread_system_time = timer_grab(&thread->system_timer); 439 mt_fixed_thread_counts(thread, counts); 440 KDBG_RELEASE(DBG_MT_INSTRS_CYCLES_THR_EXIT, 441 #ifdef MT_CORE_INSTRS 442 counts[MT_CORE_INSTRS], 443 #else /* defined(MT_CORE_INSTRS) */ 444 0, 445 #endif/* !defined(MT_CORE_INSTRS) */ 446 counts[MT_CORE_CYCLES], 447 thread_system_time, thread_user_time); 448 } 449 #endif/* MONOTONIC */ 450 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE_PID, dbg_arg1, dbg_arg2); 451 } 452 453 /* 454 * After this subtraction, this thread should never access 455 * task->bsd_info unless it got 0 back from the os_atomic_dec. It 456 * could be racing with other threads to be the last thread in the 457 * process, and the last thread in the process will tear down the proc 458 * structure and zero-out task->bsd_info. 459 */ 460 threadcnt = os_atomic_dec(&task->active_thread_count, relaxed); 461 462 #if CONFIG_COALITIONS 463 /* 464 * Leave the coalitions when last thread of task is exiting and the 465 * task is not a corpse. 466 */ 467 if (threadcnt == 0 && !task->corpse_info) { 468 coalitions_remove_task(task); 469 } 470 #endif 471 472 /* 473 * If we are the last thread to terminate and the task is 474 * associated with a BSD process, perform BSD process exit. 475 */ 476 if (threadcnt == 0 && task->bsd_info != NULL && !task_is_exec_copy(task)) { 477 mach_exception_data_type_t subcode = 0; 478 if (kdebug_enable) { 479 /* since we're the last thread in this process, trace out the command name too */ 480 long args[4] = {}; 481 kdbg_trace_string(thread->task->bsd_info, &args[0], &args[1], &args[2], &args[3]); 482 #if MONOTONIC 483 if (kdebug_debugid_enabled(DBG_MT_INSTRS_CYCLES_PROC_EXIT)) { 484 uint64_t counts[MT_CORE_NFIXED]; 485 uint64_t task_user_time; 486 uint64_t task_system_time; 487 mt_fixed_task_counts(task, counts); 488 /* since the thread time is not yet added to the task */ 489 task_user_time = task->total_user_time + timer_grab(&thread->user_timer); 490 task_system_time = task->total_system_time + timer_grab(&thread->system_timer); 491 KDBG_RELEASE((DBG_MT_INSTRS_CYCLES_PROC_EXIT), 492 #ifdef MT_CORE_INSTRS 493 counts[MT_CORE_INSTRS], 494 #else /* defined(MT_CORE_INSTRS) */ 495 0, 496 #endif/* !defined(MT_CORE_INSTRS) */ 497 counts[MT_CORE_CYCLES], 498 task_system_time, task_user_time); 499 } 500 #endif/* MONOTONIC */ 501 KDBG_RELEASE(TRACE_STRING_PROC_EXIT, args[0], args[1], args[2], args[3]); 502 } 503 504 /* Get the exit reason before proc_exit */ 505 subcode = proc_encode_exit_exception_code(task->bsd_info); 506 proc_exit(task->bsd_info); 507 /* 508 * if there is crash info in task 509 * then do the deliver action since this is 510 * last thread for this task. 511 */ 512 if (task->corpse_info) { 513 task_deliver_crash_notification(task, current_thread(), EXC_RESOURCE, subcode); 514 } 515 } 516 517 if (threadcnt == 0) { 518 task_lock(task); 519 if (task_is_a_corpse_fork(task)) { 520 thread_wakeup((event_t)&task->active_thread_count); 521 } 522 task_unlock(task); 523 } 524 525 uthread_cred_free(thread->uthread); 526 527 s = splsched(); 528 thread_lock(thread); 529 530 /* 531 * Ensure that the depress timer is no longer enqueued, 532 * so the timer (stored in the thread) can be safely deallocated 533 * 534 * TODO: build timer_call_cancel_wait 535 */ 536 537 assert((thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) == 0); 538 539 uint32_t delay_us = 1; 540 541 while (thread->depress_timer_active > 0) { 542 thread_unlock(thread); 543 splx(s); 544 545 delay(delay_us++); 546 547 if (delay_us > USEC_PER_SEC) { 548 panic("depress timer failed to inactivate!" 549 "thread: %p depress_timer_active: %d", 550 thread, thread->depress_timer_active); 551 } 552 553 s = splsched(); 554 thread_lock(thread); 555 } 556 557 /* 558 * Cancel wait timer, and wait for 559 * concurrent expirations. 560 */ 561 if (thread->wait_timer_is_set) { 562 thread->wait_timer_is_set = FALSE; 563 564 if (timer_call_cancel(&thread->wait_timer)) { 565 thread->wait_timer_active--; 566 } 567 } 568 569 delay_us = 1; 570 571 while (thread->wait_timer_active > 0) { 572 thread_unlock(thread); 573 splx(s); 574 575 delay(delay_us++); 576 577 if (delay_us > USEC_PER_SEC) { 578 panic("wait timer failed to inactivate!" 579 "thread: %p wait_timer_active: %d", 580 thread, thread->wait_timer_active); 581 } 582 583 s = splsched(); 584 thread_lock(thread); 585 } 586 587 /* 588 * If there is a reserved stack, release it. 589 */ 590 if (thread->reserved_stack != 0) { 591 stack_free_reserved(thread); 592 thread->reserved_stack = 0; 593 } 594 595 /* 596 * Mark thread as terminating, and block. 597 */ 598 thread->state |= TH_TERMINATE; 599 thread_mark_wait_locked(thread, THREAD_UNINT); 600 601 assert((thread->sched_flags & TH_SFLAG_WAITQ_PROMOTED) == 0); 602 assert((thread->sched_flags & TH_SFLAG_RW_PROMOTED) == 0); 603 assert((thread->sched_flags & TH_SFLAG_EXEC_PROMOTED) == 0); 604 assert((thread->sched_flags & TH_SFLAG_PROMOTED) == 0); 605 assert((thread->sched_flags & TH_SFLAG_THREAD_GROUP_AUTO_JOIN) == 0); 606 assert(thread->th_work_interval_flags == TH_WORK_INTERVAL_FLAGS_NONE); 607 assert(thread->kern_promotion_schedpri == 0); 608 assert(thread->waiting_for_mutex == NULL); 609 assert(thread->rwlock_count == 0); 610 assert(thread->handoff_thread == THREAD_NULL); 611 assert(thread->th_work_interval == NULL); 612 613 thread_unlock(thread); 614 /* splsched */ 615 616 thread_block((thread_continue_t)thread_terminate_continue); 617 /*NOTREACHED*/ 618 } 619 620 static bool 621 thread_ref_release(thread_t thread) 622 { 623 if (thread == THREAD_NULL) { 624 return false; 625 } 626 627 assert_thread_magic(thread); 628 629 return os_ref_release(&thread->ref_count) == 0; 630 } 631 632 /* Drop a thread refcount safely without triggering a zfree */ 633 void 634 thread_deallocate_safe(thread_t thread) 635 { 636 if (__improbable(thread_ref_release(thread))) { 637 /* enqueue the thread for thread deallocate deamon to call thread_deallocate_complete */ 638 thread_deallocate_enqueue(thread); 639 } 640 } 641 642 void 643 thread_deallocate(thread_t thread) 644 { 645 if (__improbable(thread_ref_release(thread))) { 646 thread_deallocate_complete(thread); 647 } 648 } 649 650 void 651 thread_deallocate_complete( 652 thread_t thread) 653 { 654 task_t task; 655 656 assert_thread_magic(thread); 657 658 assert(os_ref_get_count(&thread->ref_count) == 0); 659 660 if (!(thread->state & TH_TERMINATE2)) { 661 panic("thread_deallocate: thread not properly terminated\n"); 662 } 663 664 assert(thread->runq == PROCESSOR_NULL); 665 666 #if KPC 667 kpc_thread_destroy(thread); 668 #endif 669 670 ipc_thread_terminate(thread); 671 672 proc_thread_qos_deallocate(thread); 673 674 task = thread->task; 675 676 #ifdef MACH_BSD 677 { 678 void *ut = thread->uthread; 679 680 thread->uthread = NULL; 681 uthread_zone_free(ut); 682 } 683 #endif /* MACH_BSD */ 684 685 if (thread->t_ledger) { 686 ledger_dereference(thread->t_ledger); 687 } 688 if (thread->t_threadledger) { 689 ledger_dereference(thread->t_threadledger); 690 } 691 692 assert(thread->turnstile != TURNSTILE_NULL); 693 if (thread->turnstile) { 694 turnstile_deallocate(thread->turnstile); 695 } 696 697 if (IPC_VOUCHER_NULL != thread->ith_voucher) { 698 ipc_voucher_release(thread->ith_voucher); 699 } 700 701 if (thread->thread_io_stats) { 702 kheap_free(KHEAP_DATA_BUFFERS, thread->thread_io_stats, 703 sizeof(struct io_stat_info)); 704 } 705 706 if (thread->kernel_stack != 0) { 707 stack_free(thread); 708 } 709 710 lck_mtx_destroy(&thread->mutex, &thread_lck_grp); 711 machine_thread_destroy(thread); 712 713 task_deallocate(task); 714 715 #if MACH_ASSERT 716 assert_thread_magic(thread); 717 thread->thread_magic = 0; 718 #endif /* MACH_ASSERT */ 719 720 lck_mtx_lock(&tasks_threads_lock); 721 assert(terminated_threads_count > 0); 722 queue_remove(&terminated_threads, thread, thread_t, threads); 723 terminated_threads_count--; 724 lck_mtx_unlock(&tasks_threads_lock); 725 726 zfree(thread_zone, thread); 727 } 728 729 /* 730 * thread_inspect_deallocate: 731 * 732 * Drop a thread inspection reference. 733 */ 734 void 735 thread_inspect_deallocate( 736 thread_inspect_t thread_inspect) 737 { 738 return thread_deallocate((thread_t)thread_inspect); 739 } 740 741 /* 742 * thread_read_deallocate: 743 * 744 * Drop a reference on thread read port. 745 */ 746 void 747 thread_read_deallocate( 748 thread_read_t thread_read) 749 { 750 return thread_deallocate((thread_t)thread_read); 751 } 752 753 754 /* 755 * thread_exception_queue_invoke: 756 * 757 * Deliver EXC_{RESOURCE,GUARD} exception 758 */ 759 static void 760 thread_exception_queue_invoke(mpsc_queue_chain_t elm, 761 __assert_only mpsc_daemon_queue_t dq) 762 { 763 struct thread_exception_elt *elt; 764 task_t task; 765 thread_t thread; 766 exception_type_t etype; 767 768 assert(dq == &thread_exception_queue); 769 elt = mpsc_queue_element(elm, struct thread_exception_elt, link); 770 771 etype = elt->exception_type; 772 task = elt->exception_task; 773 thread = elt->exception_thread; 774 assert_thread_magic(thread); 775 776 kfree(elt, sizeof(*elt)); 777 778 /* wait for all the threads in the task to terminate */ 779 task_lock(task); 780 task_wait_till_threads_terminate_locked(task); 781 task_unlock(task); 782 783 /* Consumes the task ref returned by task_generate_corpse_internal */ 784 task_deallocate(task); 785 /* Consumes the thread ref returned by task_generate_corpse_internal */ 786 thread_deallocate(thread); 787 788 /* Deliver the notification, also clears the corpse. */ 789 task_deliver_crash_notification(task, thread, etype, 0); 790 } 791 792 /* 793 * thread_exception_enqueue: 794 * 795 * Enqueue a corpse port to be delivered an EXC_{RESOURCE,GUARD}. 796 */ 797 void 798 thread_exception_enqueue( 799 task_t task, 800 thread_t thread, 801 exception_type_t etype) 802 { 803 assert(EXC_RESOURCE == etype || EXC_GUARD == etype); 804 struct thread_exception_elt *elt = kalloc(sizeof(*elt)); 805 elt->exception_type = etype; 806 elt->exception_task = task; 807 elt->exception_thread = thread; 808 809 mpsc_daemon_enqueue(&thread_exception_queue, &elt->link, 810 MPSC_QUEUE_DISABLE_PREEMPTION); 811 } 812 813 /* 814 * thread_copy_resource_info 815 * 816 * Copy the resource info counters from source 817 * thread to destination thread. 818 */ 819 void 820 thread_copy_resource_info( 821 thread_t dst_thread, 822 thread_t src_thread) 823 { 824 dst_thread->c_switch = src_thread->c_switch; 825 dst_thread->p_switch = src_thread->p_switch; 826 dst_thread->ps_switch = src_thread->ps_switch; 827 dst_thread->precise_user_kernel_time = src_thread->precise_user_kernel_time; 828 dst_thread->user_timer = src_thread->user_timer; 829 dst_thread->user_timer_save = src_thread->user_timer_save; 830 dst_thread->system_timer = src_thread->system_timer; 831 dst_thread->system_timer_save = src_thread->system_timer_save; 832 dst_thread->runnable_timer = src_thread->runnable_timer; 833 dst_thread->vtimer_user_save = src_thread->vtimer_user_save; 834 dst_thread->vtimer_prof_save = src_thread->vtimer_prof_save; 835 dst_thread->vtimer_rlim_save = src_thread->vtimer_rlim_save; 836 dst_thread->vtimer_qos_save = src_thread->vtimer_qos_save; 837 dst_thread->syscalls_unix = src_thread->syscalls_unix; 838 dst_thread->syscalls_mach = src_thread->syscalls_mach; 839 ledger_rollup(dst_thread->t_threadledger, src_thread->t_threadledger); 840 *dst_thread->thread_io_stats = *src_thread->thread_io_stats; 841 } 842 843 static void 844 thread_terminate_queue_invoke(mpsc_queue_chain_t e, 845 __assert_only mpsc_daemon_queue_t dq) 846 { 847 thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links); 848 task_t task = thread->task; 849 850 assert(dq == &thread_terminate_queue); 851 852 task_lock(task); 853 854 /* 855 * if marked for crash reporting, skip reaping. 856 * The corpse delivery thread will clear bit and enqueue 857 * for reaping when done 858 * 859 * Note: the inspection field is set under the task lock 860 * 861 * FIXME[mad]: why enqueue for termination before `inspection` is false ? 862 */ 863 if (__improbable(thread->inspection)) { 864 simple_lock(&crashed_threads_lock, &thread_lck_grp); 865 task_unlock(task); 866 867 enqueue_tail(&crashed_threads_queue, &thread->runq_links); 868 simple_unlock(&crashed_threads_lock); 869 return; 870 } 871 872 873 task->total_user_time += timer_grab(&thread->user_timer); 874 task->total_ptime += timer_grab(&thread->ptime); 875 task->total_runnable_time += timer_grab(&thread->runnable_timer); 876 if (thread->precise_user_kernel_time) { 877 task->total_system_time += timer_grab(&thread->system_timer); 878 } else { 879 task->total_user_time += timer_grab(&thread->system_timer); 880 } 881 882 task->c_switch += thread->c_switch; 883 task->p_switch += thread->p_switch; 884 task->ps_switch += thread->ps_switch; 885 886 task->syscalls_unix += thread->syscalls_unix; 887 task->syscalls_mach += thread->syscalls_mach; 888 889 task->task_timer_wakeups_bin_1 += thread->thread_timer_wakeups_bin_1; 890 task->task_timer_wakeups_bin_2 += thread->thread_timer_wakeups_bin_2; 891 task->task_gpu_ns += ml_gpu_stat(thread); 892 task->task_energy += ml_energy_stat(thread); 893 task->decompressions += thread->decompressions; 894 895 #if MONOTONIC 896 mt_terminate_update(task, thread); 897 #endif /* MONOTONIC */ 898 899 thread_update_qos_cpu_time(thread); 900 901 queue_remove(&task->threads, thread, thread_t, task_threads); 902 task->thread_count--; 903 904 /* 905 * If the task is being halted, and there is only one thread 906 * left in the task after this one, then wakeup that thread. 907 */ 908 if (task->thread_count == 1 && task->halting) { 909 thread_wakeup((event_t)&task->halting); 910 } 911 912 task_unlock(task); 913 914 lck_mtx_lock(&tasks_threads_lock); 915 queue_remove(&threads, thread, thread_t, threads); 916 threads_count--; 917 queue_enter(&terminated_threads, thread, thread_t, threads); 918 terminated_threads_count++; 919 lck_mtx_unlock(&tasks_threads_lock); 920 921 thread_deallocate(thread); 922 } 923 924 static void 925 thread_deallocate_queue_invoke(mpsc_queue_chain_t e, 926 __assert_only mpsc_daemon_queue_t dq) 927 { 928 thread_t thread = mpsc_queue_element(e, struct thread, mpsc_links); 929 930 assert(dq == &thread_deallocate_queue); 931 932 thread_deallocate_complete(thread); 933 } 934 935 /* 936 * thread_terminate_enqueue: 937 * 938 * Enqueue a terminating thread for final disposition. 939 * 940 * Called at splsched. 941 */ 942 void 943 thread_terminate_enqueue( 944 thread_t thread) 945 { 946 KDBG_RELEASE(TRACE_DATA_THREAD_TERMINATE, thread->thread_id); 947 948 mpsc_daemon_enqueue(&thread_terminate_queue, &thread->mpsc_links, 949 MPSC_QUEUE_DISABLE_PREEMPTION); 950 } 951 952 /* 953 * thread_deallocate_enqueue: 954 * 955 * Enqueue a thread for final deallocation. 956 */ 957 static void 958 thread_deallocate_enqueue( 959 thread_t thread) 960 { 961 mpsc_daemon_enqueue(&thread_deallocate_queue, &thread->mpsc_links, 962 MPSC_QUEUE_DISABLE_PREEMPTION); 963 } 964 965 /* 966 * thread_terminate_crashed_threads: 967 * walk the list of crashed threads and put back set of threads 968 * who are no longer being inspected. 969 */ 970 void 971 thread_terminate_crashed_threads(void) 972 { 973 thread_t th_remove; 974 975 simple_lock(&crashed_threads_lock, &thread_lck_grp); 976 /* 977 * loop through the crashed threads queue 978 * to put any threads that are not being inspected anymore 979 */ 980 981 qe_foreach_element_safe(th_remove, &crashed_threads_queue, runq_links) { 982 /* make sure current_thread is never in crashed queue */ 983 assert(th_remove != current_thread()); 984 985 if (th_remove->inspection == FALSE) { 986 remqueue(&th_remove->runq_links); 987 mpsc_daemon_enqueue(&thread_terminate_queue, &th_remove->mpsc_links, 988 MPSC_QUEUE_NONE); 989 } 990 } 991 992 simple_unlock(&crashed_threads_lock); 993 } 994 995 /* 996 * thread_stack_queue_invoke: 997 * 998 * Perform stack allocation as required due to 999 * invoke failures. 1000 */ 1001 static void 1002 thread_stack_queue_invoke(mpsc_queue_chain_t elm, 1003 __assert_only mpsc_daemon_queue_t dq) 1004 { 1005 thread_t thread = mpsc_queue_element(elm, struct thread, mpsc_links); 1006 1007 assert(dq == &thread_stack_queue); 1008 1009 /* allocate stack with interrupts enabled so that we can call into VM */ 1010 stack_alloc(thread); 1011 1012 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_END, thread_tid(thread), 0, 0, 0, 0); 1013 1014 spl_t s = splsched(); 1015 thread_lock(thread); 1016 thread_setrun(thread, SCHED_PREEMPT | SCHED_TAILQ); 1017 thread_unlock(thread); 1018 splx(s); 1019 } 1020 1021 /* 1022 * thread_stack_enqueue: 1023 * 1024 * Enqueue a thread for stack allocation. 1025 * 1026 * Called at splsched. 1027 */ 1028 void 1029 thread_stack_enqueue( 1030 thread_t thread) 1031 { 1032 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED, MACH_STACK_WAIT) | DBG_FUNC_START, thread_tid(thread), 0, 0, 0, 0); 1033 assert_thread_magic(thread); 1034 1035 mpsc_daemon_enqueue(&thread_stack_queue, &thread->mpsc_links, 1036 MPSC_QUEUE_DISABLE_PREEMPTION); 1037 } 1038 1039 void 1040 thread_daemon_init(void) 1041 { 1042 kern_return_t result; 1043 1044 thread_deallocate_daemon_init(); 1045 1046 thread_deallocate_daemon_register_queue(&thread_terminate_queue, 1047 thread_terminate_queue_invoke); 1048 1049 thread_deallocate_daemon_register_queue(&thread_deallocate_queue, 1050 thread_deallocate_queue_invoke); 1051 1052 simple_lock_init(&crashed_threads_lock, 0); 1053 queue_init(&crashed_threads_queue); 1054 1055 result = mpsc_daemon_queue_init_with_thread(&thread_stack_queue, 1056 thread_stack_queue_invoke, BASEPRI_PREEMPT_HIGH, 1057 "daemon.thread-stack"); 1058 if (result != KERN_SUCCESS) { 1059 panic("thread_daemon_init: thread_stack_daemon"); 1060 } 1061 1062 result = mpsc_daemon_queue_init_with_thread(&thread_exception_queue, 1063 thread_exception_queue_invoke, MINPRI_KERNEL, 1064 "daemon.thread-exception"); 1065 if (result != KERN_SUCCESS) { 1066 panic("thread_daemon_init: thread_exception_daemon"); 1067 } 1068 } 1069 1070 __options_decl(thread_create_internal_options_t, uint32_t, { 1071 TH_OPTION_NONE = 0x00, 1072 TH_OPTION_NOCRED = 0x01, 1073 TH_OPTION_NOSUSP = 0x02, 1074 TH_OPTION_WORKQ = 0x04, 1075 TH_OPTION_IMMOVABLE = 0x08, 1076 TH_OPTION_PINNED = 0x10, 1077 }); 1078 1079 /* 1080 * Create a new thread. 1081 * Doesn't start the thread running. 1082 * 1083 * Task and tasks_threads_lock are returned locked on success. 1084 */ 1085 static kern_return_t 1086 thread_create_internal( 1087 task_t parent_task, 1088 integer_t priority, 1089 thread_continue_t continuation, 1090 void *parameter, 1091 thread_create_internal_options_t options, 1092 thread_t *out_thread) 1093 { 1094 thread_t new_thread; 1095 static thread_t first_thread; 1096 ipc_thread_init_options_t init_options = IPC_THREAD_INIT_NONE; 1097 1098 /* 1099 * Allocate a thread and initialize static fields 1100 */ 1101 if (first_thread == THREAD_NULL) { 1102 new_thread = first_thread = current_thread(); 1103 } else { 1104 new_thread = (thread_t)zalloc(thread_zone); 1105 } 1106 if (new_thread == THREAD_NULL) { 1107 return KERN_RESOURCE_SHORTAGE; 1108 } 1109 1110 if (new_thread != first_thread) { 1111 init_thread_from_template(new_thread); 1112 } 1113 1114 if (options & TH_OPTION_PINNED) { 1115 init_options |= IPC_THREAD_INIT_PINNED; 1116 } 1117 1118 if (options & TH_OPTION_IMMOVABLE) { 1119 init_options |= IPC_THREAD_INIT_IMMOVABLE; 1120 } 1121 1122 os_ref_init_count(&new_thread->ref_count, &thread_refgrp, 2); 1123 #if DEBUG || DEVELOPMENT 1124 queue_init(&new_thread->t_temp_alloc_list); 1125 #endif /* DEBUG || DEVELOPMENT */ 1126 1127 #ifdef MACH_BSD 1128 new_thread->uthread = uthread_alloc(parent_task, new_thread, (options & TH_OPTION_NOCRED) != 0); 1129 if (new_thread->uthread == NULL) { 1130 #if MACH_ASSERT 1131 new_thread->thread_magic = 0; 1132 #endif /* MACH_ASSERT */ 1133 1134 zfree(thread_zone, new_thread); 1135 return KERN_RESOURCE_SHORTAGE; 1136 } 1137 #endif /* MACH_BSD */ 1138 1139 if (machine_thread_create(new_thread, parent_task) != KERN_SUCCESS) { 1140 #ifdef MACH_BSD 1141 void *ut = new_thread->uthread; 1142 1143 new_thread->uthread = NULL; 1144 /* cred free may not be necessary */ 1145 uthread_cleanup(parent_task, ut, parent_task->bsd_info); 1146 uthread_cred_free(ut); 1147 uthread_zone_free(ut); 1148 #endif /* MACH_BSD */ 1149 1150 #if MACH_ASSERT 1151 new_thread->thread_magic = 0; 1152 #endif /* MACH_ASSERT */ 1153 1154 zfree(thread_zone, new_thread); 1155 return KERN_FAILURE; 1156 } 1157 1158 new_thread->task = parent_task; 1159 1160 thread_lock_init(new_thread); 1161 wake_lock_init(new_thread); 1162 1163 lck_mtx_init(&new_thread->mutex, &thread_lck_grp, LCK_ATTR_NULL); 1164 1165 ipc_thread_init(new_thread, init_options); 1166 1167 new_thread->continuation = continuation; 1168 new_thread->parameter = parameter; 1169 new_thread->inheritor_flags = TURNSTILE_UPDATE_FLAGS_NONE; 1170 priority_queue_init(&new_thread->sched_inheritor_queue); 1171 priority_queue_init(&new_thread->base_inheritor_queue); 1172 #if CONFIG_SCHED_CLUTCH 1173 priority_queue_entry_init(&new_thread->th_clutch_runq_link); 1174 priority_queue_entry_init(&new_thread->th_clutch_pri_link); 1175 #endif /* CONFIG_SCHED_CLUTCH */ 1176 1177 #if CONFIG_SCHED_EDGE 1178 new_thread->th_bound_cluster_enqueued = false; 1179 #endif /* CONFIG_SCHED_EDGE */ 1180 1181 /* Allocate I/O Statistics structure */ 1182 new_thread->thread_io_stats = kheap_alloc(KHEAP_DATA_BUFFERS, 1183 sizeof(struct io_stat_info), Z_WAITOK | Z_ZERO); 1184 assert(new_thread->thread_io_stats != NULL); 1185 1186 #if KASAN 1187 kasan_init_thread(&new_thread->kasan_data); 1188 #endif 1189 1190 #if CONFIG_KSANCOV 1191 new_thread->ksancov_data = NULL; 1192 #endif 1193 1194 #if CONFIG_IOSCHED 1195 /* Clear out the I/O Scheduling info for AppleFSCompression */ 1196 new_thread->decmp_upl = NULL; 1197 #endif /* CONFIG_IOSCHED */ 1198 1199 new_thread->thread_region_page_shift = 0; 1200 1201 #if DEVELOPMENT || DEBUG 1202 task_lock(parent_task); 1203 uint16_t thread_limit = parent_task->task_thread_limit; 1204 if (exc_resource_threads_enabled && 1205 thread_limit > 0 && 1206 parent_task->thread_count >= thread_limit && 1207 !parent_task->task_has_crossed_thread_limit && 1208 !(parent_task->t_flags & TF_CORPSE)) { 1209 int thread_count = parent_task->thread_count; 1210 parent_task->task_has_crossed_thread_limit = TRUE; 1211 task_unlock(parent_task); 1212 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(parent_task, thread_count); 1213 } else { 1214 task_unlock(parent_task); 1215 } 1216 #endif 1217 1218 lck_mtx_lock(&tasks_threads_lock); 1219 task_lock(parent_task); 1220 1221 /* 1222 * Fail thread creation if parent task is being torn down or has too many threads 1223 * If the caller asked for TH_OPTION_NOSUSP, also fail if the parent task is suspended 1224 */ 1225 if (parent_task->active == 0 || parent_task->halting || 1226 (parent_task->suspend_count > 0 && (options & TH_OPTION_NOSUSP) != 0) || 1227 (parent_task->thread_count >= task_threadmax && parent_task != kernel_task)) { 1228 task_unlock(parent_task); 1229 lck_mtx_unlock(&tasks_threads_lock); 1230 1231 #ifdef MACH_BSD 1232 { 1233 void *ut = new_thread->uthread; 1234 1235 new_thread->uthread = NULL; 1236 uthread_cleanup(parent_task, ut, parent_task->bsd_info); 1237 /* cred free may not be necessary */ 1238 uthread_cred_free(ut); 1239 uthread_zone_free(ut); 1240 } 1241 #endif /* MACH_BSD */ 1242 ipc_thread_disable(new_thread); 1243 ipc_thread_terminate(new_thread); 1244 kheap_free(KHEAP_DATA_BUFFERS, new_thread->thread_io_stats, 1245 sizeof(struct io_stat_info)); 1246 lck_mtx_destroy(&new_thread->mutex, &thread_lck_grp); 1247 machine_thread_destroy(new_thread); 1248 zfree(thread_zone, new_thread); 1249 return KERN_FAILURE; 1250 } 1251 1252 /* Protected by the tasks_threads_lock */ 1253 new_thread->thread_id = ++thread_unique_id; 1254 1255 /* New threads inherit any default state on the task */ 1256 machine_thread_inherit_taskwide(new_thread, parent_task); 1257 1258 task_reference_internal(parent_task); 1259 1260 if (new_thread->task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_PERTHR_LIMIT) { 1261 /* 1262 * This task has a per-thread CPU limit; make sure this new thread 1263 * gets its limit set too, before it gets out of the kernel. 1264 */ 1265 act_set_astledger(new_thread); 1266 } 1267 1268 /* Instantiate a thread ledger. Do not fail thread creation if ledger creation fails. */ 1269 if ((new_thread->t_threadledger = ledger_instantiate(thread_ledger_template, 1270 LEDGER_CREATE_INACTIVE_ENTRIES)) != LEDGER_NULL) { 1271 ledger_entry_setactive(new_thread->t_threadledger, thread_ledgers.cpu_time); 1272 } 1273 1274 new_thread->t_bankledger = LEDGER_NULL; 1275 new_thread->t_deduct_bank_ledger_time = 0; 1276 new_thread->t_deduct_bank_ledger_energy = 0; 1277 1278 new_thread->t_ledger = new_thread->task->ledger; 1279 if (new_thread->t_ledger) { 1280 ledger_reference(new_thread->t_ledger); 1281 } 1282 1283 #if defined(CONFIG_SCHED_MULTIQ) 1284 /* Cache the task's sched_group */ 1285 new_thread->sched_group = parent_task->sched_group; 1286 #endif /* defined(CONFIG_SCHED_MULTIQ) */ 1287 1288 /* Cache the task's map */ 1289 new_thread->map = parent_task->map; 1290 1291 timer_call_setup(&new_thread->wait_timer, thread_timer_expire, new_thread); 1292 timer_call_setup(&new_thread->depress_timer, thread_depress_expire, new_thread); 1293 1294 #if KPC 1295 kpc_thread_create(new_thread); 1296 #endif 1297 1298 /* Set the thread's scheduling parameters */ 1299 new_thread->sched_mode = SCHED(initial_thread_sched_mode)(parent_task); 1300 new_thread->max_priority = parent_task->max_priority; 1301 new_thread->task_priority = parent_task->priority; 1302 1303 #if CONFIG_THREAD_GROUPS 1304 thread_group_init_thread(new_thread, parent_task); 1305 #endif /* CONFIG_THREAD_GROUPS */ 1306 1307 int new_priority = (priority < 0) ? parent_task->priority: priority; 1308 new_priority = (priority < 0)? parent_task->priority: priority; 1309 if (new_priority > new_thread->max_priority) { 1310 new_priority = new_thread->max_priority; 1311 } 1312 #if !defined(XNU_TARGET_OS_OSX) 1313 if (new_priority < MAXPRI_THROTTLE) { 1314 new_priority = MAXPRI_THROTTLE; 1315 } 1316 #endif /* !defined(XNU_TARGET_OS_OSX) */ 1317 1318 new_thread->importance = new_priority - new_thread->task_priority; 1319 1320 sched_set_thread_base_priority(new_thread, new_priority); 1321 1322 #if defined(CONFIG_SCHED_TIMESHARE_CORE) 1323 new_thread->sched_stamp = sched_tick; 1324 #if CONFIG_SCHED_CLUTCH 1325 new_thread->pri_shift = sched_clutch_thread_pri_shift(new_thread, new_thread->th_sched_bucket); 1326 #else /* CONFIG_SCHED_CLUTCH */ 1327 new_thread->pri_shift = sched_pri_shifts[new_thread->th_sched_bucket]; 1328 #endif /* CONFIG_SCHED_CLUTCH */ 1329 #endif /* defined(CONFIG_SCHED_TIMESHARE_CORE) */ 1330 1331 if (parent_task->max_priority <= MAXPRI_THROTTLE) { 1332 sched_thread_mode_demote(new_thread, TH_SFLAG_THROTTLED); 1333 } 1334 1335 thread_policy_create(new_thread); 1336 1337 /* Chain the thread onto the task's list */ 1338 queue_enter(&parent_task->threads, new_thread, thread_t, task_threads); 1339 parent_task->thread_count++; 1340 1341 /* So terminating threads don't need to take the task lock to decrement */ 1342 os_atomic_inc(&parent_task->active_thread_count, relaxed); 1343 1344 queue_enter(&threads, new_thread, thread_t, threads); 1345 threads_count++; 1346 1347 new_thread->active = TRUE; 1348 if (task_is_a_corpse_fork(parent_task)) { 1349 /* Set the inspection bit if the task is a corpse fork */ 1350 new_thread->inspection = TRUE; 1351 } else { 1352 new_thread->inspection = FALSE; 1353 } 1354 new_thread->corpse_dup = FALSE; 1355 new_thread->turnstile = turnstile_alloc(); 1356 1357 1358 *out_thread = new_thread; 1359 1360 if (kdebug_enable) { 1361 long args[4] = {}; 1362 1363 kdbg_trace_data(parent_task->bsd_info, &args[1], &args[3]); 1364 1365 /* 1366 * Starting with 26604425, exec'ing creates a new task/thread. 1367 * 1368 * NEWTHREAD in the current process has two possible meanings: 1369 * 1370 * 1) Create a new thread for this process. 1371 * 2) Create a new thread for the future process this will become in an 1372 * exec. 1373 * 1374 * To disambiguate these, arg3 will be set to TRUE for case #2. 1375 * 1376 * The value we need to find (TPF_EXEC_COPY) is stable in the case of a 1377 * task exec'ing. The read of t_procflags does not take the proc_lock. 1378 */ 1379 args[2] = task_is_exec_copy(parent_task) ? 1 : 0; 1380 1381 KDBG_RELEASE(TRACE_DATA_NEWTHREAD, (uintptr_t)thread_tid(new_thread), 1382 args[1], args[2], args[3]); 1383 1384 kdbg_trace_string(parent_task->bsd_info, &args[0], &args[1], 1385 &args[2], &args[3]); 1386 KDBG_RELEASE(TRACE_STRING_NEWTHREAD, args[0], args[1], args[2], 1387 args[3]); 1388 } 1389 1390 DTRACE_PROC1(lwp__create, thread_t, *out_thread); 1391 1392 return KERN_SUCCESS; 1393 } 1394 1395 static kern_return_t 1396 thread_create_with_options_internal( 1397 task_t task, 1398 thread_t *new_thread, 1399 boolean_t from_user, 1400 thread_create_internal_options_t options, 1401 thread_continue_t continuation) 1402 { 1403 kern_return_t result; 1404 thread_t thread; 1405 1406 if (task == TASK_NULL || task == kernel_task) { 1407 return KERN_INVALID_ARGUMENT; 1408 } 1409 1410 #if CONFIG_MACF 1411 if (from_user && current_task() != task && 1412 mac_proc_check_remote_thread_create(task, -1, NULL, 0) != 0) { 1413 return KERN_DENIED; 1414 } 1415 #endif 1416 1417 result = thread_create_internal(task, -1, continuation, NULL, options, &thread); 1418 if (result != KERN_SUCCESS) { 1419 return result; 1420 } 1421 1422 thread->user_stop_count = 1; 1423 thread_hold(thread); 1424 if (task->suspend_count > 0) { 1425 thread_hold(thread); 1426 } 1427 1428 if (from_user) { 1429 extmod_statistics_incr_thread_create(task); 1430 } 1431 1432 task_unlock(task); 1433 lck_mtx_unlock(&tasks_threads_lock); 1434 1435 *new_thread = thread; 1436 1437 return KERN_SUCCESS; 1438 } 1439 1440 /* No prototype, since task_server.h has the _from_user version if KERNEL_SERVER */ 1441 kern_return_t 1442 thread_create( 1443 task_t task, 1444 thread_t *new_thread); 1445 1446 kern_return_t 1447 thread_create( 1448 task_t task, 1449 thread_t *new_thread) 1450 { 1451 return thread_create_with_options_internal(task, new_thread, FALSE, TH_OPTION_NONE, 1452 (thread_continue_t)thread_bootstrap_return); 1453 } 1454 1455 /* 1456 * Create a thread that has its itk_self pinned 1457 * Deprecated, should be cleanup once rdar://70892168 lands 1458 */ 1459 kern_return_t 1460 thread_create_pinned( 1461 task_t task, 1462 thread_t *new_thread) 1463 { 1464 return thread_create_with_options_internal(task, new_thread, FALSE, 1465 TH_OPTION_PINNED | TH_OPTION_IMMOVABLE, (thread_continue_t)thread_bootstrap_return); 1466 } 1467 1468 kern_return_t 1469 thread_create_immovable( 1470 task_t task, 1471 thread_t *new_thread) 1472 { 1473 return thread_create_with_options_internal(task, new_thread, FALSE, 1474 TH_OPTION_IMMOVABLE, (thread_continue_t)thread_bootstrap_return); 1475 } 1476 1477 kern_return_t 1478 thread_create_from_user( 1479 task_t task, 1480 thread_t *new_thread) 1481 { 1482 return thread_create_with_options_internal(task, new_thread, TRUE, TH_OPTION_NONE, 1483 (thread_continue_t)thread_bootstrap_return); 1484 } 1485 1486 kern_return_t 1487 thread_create_with_continuation( 1488 task_t task, 1489 thread_t *new_thread, 1490 thread_continue_t continuation) 1491 { 1492 return thread_create_with_options_internal(task, new_thread, FALSE, TH_OPTION_NONE, continuation); 1493 } 1494 1495 /* 1496 * Create a thread that is already started, but is waiting on an event 1497 */ 1498 static kern_return_t 1499 thread_create_waiting_internal( 1500 task_t task, 1501 thread_continue_t continuation, 1502 event_t event, 1503 block_hint_t block_hint, 1504 int options, 1505 thread_t *new_thread) 1506 { 1507 kern_return_t result; 1508 thread_t thread; 1509 1510 if (task == TASK_NULL || task == kernel_task) { 1511 return KERN_INVALID_ARGUMENT; 1512 } 1513 1514 result = thread_create_internal(task, -1, continuation, NULL, 1515 options, &thread); 1516 if (result != KERN_SUCCESS) { 1517 return result; 1518 } 1519 1520 /* note no user_stop_count or thread_hold here */ 1521 1522 if (task->suspend_count > 0) { 1523 thread_hold(thread); 1524 } 1525 1526 thread_mtx_lock(thread); 1527 thread_set_pending_block_hint(thread, block_hint); 1528 if (options & TH_OPTION_WORKQ) { 1529 thread->static_param = true; 1530 event = workq_thread_init_and_wq_lock(task, thread); 1531 } 1532 thread_start_in_assert_wait(thread, event, THREAD_INTERRUPTIBLE); 1533 thread_mtx_unlock(thread); 1534 1535 task_unlock(task); 1536 lck_mtx_unlock(&tasks_threads_lock); 1537 1538 *new_thread = thread; 1539 1540 return KERN_SUCCESS; 1541 } 1542 1543 kern_return_t 1544 thread_create_waiting( 1545 task_t task, 1546 thread_continue_t continuation, 1547 event_t event, 1548 th_create_waiting_options_t options, 1549 thread_t *new_thread) 1550 { 1551 thread_create_internal_options_t ci_options = TH_OPTION_NONE; 1552 1553 assert((options & ~TH_CREATE_WAITING_OPTION_MASK) == 0); 1554 if (options & TH_CREATE_WAITING_OPTION_PINNED) { 1555 ci_options |= TH_OPTION_PINNED; 1556 } 1557 if (options & TH_CREATE_WAITING_OPTION_IMMOVABLE) { 1558 ci_options |= TH_OPTION_IMMOVABLE; 1559 } 1560 1561 return thread_create_waiting_internal(task, continuation, event, 1562 kThreadWaitNone, ci_options, new_thread); 1563 } 1564 1565 1566 static kern_return_t 1567 thread_create_running_internal2( 1568 task_t task, 1569 int flavor, 1570 thread_state_t new_state, 1571 mach_msg_type_number_t new_state_count, 1572 thread_t *new_thread, 1573 boolean_t from_user) 1574 { 1575 kern_return_t result; 1576 thread_t thread; 1577 1578 if (task == TASK_NULL || task == kernel_task) { 1579 return KERN_INVALID_ARGUMENT; 1580 } 1581 1582 #if CONFIG_MACF 1583 if (from_user && current_task() != task && 1584 mac_proc_check_remote_thread_create(task, flavor, new_state, new_state_count) != 0) { 1585 return KERN_DENIED; 1586 } 1587 #endif 1588 1589 result = thread_create_internal(task, -1, 1590 (thread_continue_t)thread_bootstrap_return, NULL, 1591 TH_OPTION_NONE, &thread); 1592 if (result != KERN_SUCCESS) { 1593 return result; 1594 } 1595 1596 if (task->suspend_count > 0) { 1597 thread_hold(thread); 1598 } 1599 1600 if (from_user) { 1601 result = machine_thread_state_convert_from_user(thread, flavor, 1602 new_state, new_state_count); 1603 } 1604 if (result == KERN_SUCCESS) { 1605 result = machine_thread_set_state(thread, flavor, new_state, 1606 new_state_count); 1607 } 1608 if (result != KERN_SUCCESS) { 1609 task_unlock(task); 1610 lck_mtx_unlock(&tasks_threads_lock); 1611 1612 thread_terminate(thread); 1613 thread_deallocate(thread); 1614 return result; 1615 } 1616 1617 thread_mtx_lock(thread); 1618 thread_start(thread); 1619 thread_mtx_unlock(thread); 1620 1621 if (from_user) { 1622 extmod_statistics_incr_thread_create(task); 1623 } 1624 1625 task_unlock(task); 1626 lck_mtx_unlock(&tasks_threads_lock); 1627 1628 *new_thread = thread; 1629 1630 return result; 1631 } 1632 1633 /* Prototype, see justification above */ 1634 kern_return_t 1635 thread_create_running( 1636 task_t task, 1637 int flavor, 1638 thread_state_t new_state, 1639 mach_msg_type_number_t new_state_count, 1640 thread_t *new_thread); 1641 1642 kern_return_t 1643 thread_create_running( 1644 task_t task, 1645 int flavor, 1646 thread_state_t new_state, 1647 mach_msg_type_number_t new_state_count, 1648 thread_t *new_thread) 1649 { 1650 return thread_create_running_internal2( 1651 task, flavor, new_state, new_state_count, 1652 new_thread, FALSE); 1653 } 1654 1655 kern_return_t 1656 thread_create_running_from_user( 1657 task_t task, 1658 int flavor, 1659 thread_state_t new_state, 1660 mach_msg_type_number_t new_state_count, 1661 thread_t *new_thread) 1662 { 1663 return thread_create_running_internal2( 1664 task, flavor, new_state, new_state_count, 1665 new_thread, TRUE); 1666 } 1667 1668 kern_return_t 1669 thread_create_workq_waiting( 1670 task_t task, 1671 thread_continue_t continuation, 1672 thread_t *new_thread) 1673 { 1674 /* 1675 * Create thread, but don't pin control port just yet, in case someone calls 1676 * task_threads() and deallocates pinned port before kernel copyout happens, 1677 * which will result in pinned port guard exception. Instead, pin and make 1678 * it immovable atomically at copyout during workq_setup_and_run(). 1679 */ 1680 int options = TH_OPTION_NOCRED | TH_OPTION_NOSUSP | TH_OPTION_WORKQ | TH_OPTION_IMMOVABLE; 1681 return thread_create_waiting_internal(task, continuation, NULL, 1682 kThreadWaitParkedWorkQueue, options, new_thread); 1683 } 1684 1685 /* 1686 * kernel_thread_create: 1687 * 1688 * Create a thread in the kernel task 1689 * to execute in kernel context. 1690 */ 1691 kern_return_t 1692 kernel_thread_create( 1693 thread_continue_t continuation, 1694 void *parameter, 1695 integer_t priority, 1696 thread_t *new_thread) 1697 { 1698 kern_return_t result; 1699 thread_t thread; 1700 task_t task = kernel_task; 1701 1702 result = thread_create_internal(task, priority, continuation, parameter, 1703 TH_OPTION_NOCRED | TH_OPTION_NONE, &thread); 1704 if (result != KERN_SUCCESS) { 1705 return result; 1706 } 1707 1708 task_unlock(task); 1709 lck_mtx_unlock(&tasks_threads_lock); 1710 1711 stack_alloc(thread); 1712 assert(thread->kernel_stack != 0); 1713 #if !defined(XNU_TARGET_OS_OSX) 1714 if (priority > BASEPRI_KERNEL) 1715 #endif 1716 thread->reserved_stack = thread->kernel_stack; 1717 1718 if (debug_task & 1) { 1719 kprintf("kernel_thread_create: thread = %p continuation = %p\n", thread, continuation); 1720 } 1721 *new_thread = thread; 1722 1723 return result; 1724 } 1725 1726 kern_return_t 1727 kernel_thread_start_priority( 1728 thread_continue_t continuation, 1729 void *parameter, 1730 integer_t priority, 1731 thread_t *new_thread) 1732 { 1733 kern_return_t result; 1734 thread_t thread; 1735 1736 result = kernel_thread_create(continuation, parameter, priority, &thread); 1737 if (result != KERN_SUCCESS) { 1738 return result; 1739 } 1740 1741 *new_thread = thread; 1742 1743 thread_mtx_lock(thread); 1744 thread_start(thread); 1745 thread_mtx_unlock(thread); 1746 1747 return result; 1748 } 1749 1750 kern_return_t 1751 kernel_thread_start( 1752 thread_continue_t continuation, 1753 void *parameter, 1754 thread_t *new_thread) 1755 { 1756 return kernel_thread_start_priority(continuation, parameter, -1, new_thread); 1757 } 1758 1759 /* Separated into helper function so it can be used by THREAD_BASIC_INFO and THREAD_EXTENDED_INFO */ 1760 /* it is assumed that the thread is locked by the caller */ 1761 static void 1762 retrieve_thread_basic_info(thread_t thread, thread_basic_info_t basic_info) 1763 { 1764 int state, flags; 1765 1766 /* fill in info */ 1767 1768 thread_read_times(thread, &basic_info->user_time, 1769 &basic_info->system_time, NULL); 1770 1771 /* 1772 * Update lazy-evaluated scheduler info because someone wants it. 1773 */ 1774 if (SCHED(can_update_priority)(thread)) { 1775 SCHED(update_priority)(thread); 1776 } 1777 1778 basic_info->sleep_time = 0; 1779 1780 /* 1781 * To calculate cpu_usage, first correct for timer rate, 1782 * then for 5/8 ageing. The correction factor [3/5] is 1783 * (1/(5/8) - 1). 1784 */ 1785 basic_info->cpu_usage = 0; 1786 #if defined(CONFIG_SCHED_TIMESHARE_CORE) 1787 if (sched_tick_interval) { 1788 basic_info->cpu_usage = (integer_t)(((uint64_t)thread->cpu_usage 1789 * TH_USAGE_SCALE) / sched_tick_interval); 1790 basic_info->cpu_usage = (basic_info->cpu_usage * 3) / 5; 1791 } 1792 #endif 1793 1794 if (basic_info->cpu_usage > TH_USAGE_SCALE) { 1795 basic_info->cpu_usage = TH_USAGE_SCALE; 1796 } 1797 1798 basic_info->policy = ((thread->sched_mode == TH_MODE_TIMESHARE)? 1799 POLICY_TIMESHARE: POLICY_RR); 1800 1801 flags = 0; 1802 if (thread->options & TH_OPT_IDLE_THREAD) { 1803 flags |= TH_FLAGS_IDLE; 1804 } 1805 1806 if (thread->options & TH_OPT_GLOBAL_FORCED_IDLE) { 1807 flags |= TH_FLAGS_GLOBAL_FORCED_IDLE; 1808 } 1809 1810 if (!thread->kernel_stack) { 1811 flags |= TH_FLAGS_SWAPPED; 1812 } 1813 1814 state = 0; 1815 if (thread->state & TH_TERMINATE) { 1816 state = TH_STATE_HALTED; 1817 } else if (thread->state & TH_RUN) { 1818 state = TH_STATE_RUNNING; 1819 } else if (thread->state & TH_UNINT) { 1820 state = TH_STATE_UNINTERRUPTIBLE; 1821 } else if (thread->state & TH_SUSP) { 1822 state = TH_STATE_STOPPED; 1823 } else if (thread->state & TH_WAIT) { 1824 state = TH_STATE_WAITING; 1825 } 1826 1827 basic_info->run_state = state; 1828 basic_info->flags = flags; 1829 1830 basic_info->suspend_count = thread->user_stop_count; 1831 1832 return; 1833 } 1834 1835 kern_return_t 1836 thread_info_internal( 1837 thread_t thread, 1838 thread_flavor_t flavor, 1839 thread_info_t thread_info_out, /* ptr to OUT array */ 1840 mach_msg_type_number_t *thread_info_count) /*IN/OUT*/ 1841 { 1842 spl_t s; 1843 1844 if (thread == THREAD_NULL) { 1845 return KERN_INVALID_ARGUMENT; 1846 } 1847 1848 if (flavor == THREAD_BASIC_INFO) { 1849 if (*thread_info_count < THREAD_BASIC_INFO_COUNT) { 1850 return KERN_INVALID_ARGUMENT; 1851 } 1852 1853 s = splsched(); 1854 thread_lock(thread); 1855 1856 retrieve_thread_basic_info(thread, (thread_basic_info_t) thread_info_out); 1857 1858 thread_unlock(thread); 1859 splx(s); 1860 1861 *thread_info_count = THREAD_BASIC_INFO_COUNT; 1862 1863 return KERN_SUCCESS; 1864 } else if (flavor == THREAD_IDENTIFIER_INFO) { 1865 thread_identifier_info_t identifier_info; 1866 1867 if (*thread_info_count < THREAD_IDENTIFIER_INFO_COUNT) { 1868 return KERN_INVALID_ARGUMENT; 1869 } 1870 1871 identifier_info = __IGNORE_WCASTALIGN((thread_identifier_info_t)thread_info_out); 1872 1873 s = splsched(); 1874 thread_lock(thread); 1875 1876 identifier_info->thread_id = thread->thread_id; 1877 identifier_info->thread_handle = thread->machine.cthread_self; 1878 identifier_info->dispatch_qaddr = thread_dispatchqaddr(thread); 1879 1880 thread_unlock(thread); 1881 splx(s); 1882 return KERN_SUCCESS; 1883 } else if (flavor == THREAD_SCHED_TIMESHARE_INFO) { 1884 policy_timeshare_info_t ts_info; 1885 1886 if (*thread_info_count < POLICY_TIMESHARE_INFO_COUNT) { 1887 return KERN_INVALID_ARGUMENT; 1888 } 1889 1890 ts_info = (policy_timeshare_info_t)thread_info_out; 1891 1892 s = splsched(); 1893 thread_lock(thread); 1894 1895 if (thread->sched_mode != TH_MODE_TIMESHARE) { 1896 thread_unlock(thread); 1897 splx(s); 1898 return KERN_INVALID_POLICY; 1899 } 1900 1901 ts_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0; 1902 if (ts_info->depressed) { 1903 ts_info->base_priority = DEPRESSPRI; 1904 ts_info->depress_priority = thread->base_pri; 1905 } else { 1906 ts_info->base_priority = thread->base_pri; 1907 ts_info->depress_priority = -1; 1908 } 1909 1910 ts_info->cur_priority = thread->sched_pri; 1911 ts_info->max_priority = thread->max_priority; 1912 1913 thread_unlock(thread); 1914 splx(s); 1915 1916 *thread_info_count = POLICY_TIMESHARE_INFO_COUNT; 1917 1918 return KERN_SUCCESS; 1919 } else if (flavor == THREAD_SCHED_FIFO_INFO) { 1920 if (*thread_info_count < POLICY_FIFO_INFO_COUNT) { 1921 return KERN_INVALID_ARGUMENT; 1922 } 1923 1924 return KERN_INVALID_POLICY; 1925 } else if (flavor == THREAD_SCHED_RR_INFO) { 1926 policy_rr_info_t rr_info; 1927 uint32_t quantum_time; 1928 uint64_t quantum_ns; 1929 1930 if (*thread_info_count < POLICY_RR_INFO_COUNT) { 1931 return KERN_INVALID_ARGUMENT; 1932 } 1933 1934 rr_info = (policy_rr_info_t) thread_info_out; 1935 1936 s = splsched(); 1937 thread_lock(thread); 1938 1939 if (thread->sched_mode == TH_MODE_TIMESHARE) { 1940 thread_unlock(thread); 1941 splx(s); 1942 1943 return KERN_INVALID_POLICY; 1944 } 1945 1946 rr_info->depressed = (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) != 0; 1947 if (rr_info->depressed) { 1948 rr_info->base_priority = DEPRESSPRI; 1949 rr_info->depress_priority = thread->base_pri; 1950 } else { 1951 rr_info->base_priority = thread->base_pri; 1952 rr_info->depress_priority = -1; 1953 } 1954 1955 quantum_time = SCHED(initial_quantum_size)(THREAD_NULL); 1956 absolutetime_to_nanoseconds(quantum_time, &quantum_ns); 1957 1958 rr_info->max_priority = thread->max_priority; 1959 rr_info->quantum = (uint32_t)(quantum_ns / 1000 / 1000); 1960 1961 thread_unlock(thread); 1962 splx(s); 1963 1964 *thread_info_count = POLICY_RR_INFO_COUNT; 1965 1966 return KERN_SUCCESS; 1967 } else if (flavor == THREAD_EXTENDED_INFO) { 1968 thread_basic_info_data_t basic_info; 1969 thread_extended_info_t extended_info = __IGNORE_WCASTALIGN((thread_extended_info_t)thread_info_out); 1970 1971 if (*thread_info_count < THREAD_EXTENDED_INFO_COUNT) { 1972 return KERN_INVALID_ARGUMENT; 1973 } 1974 1975 s = splsched(); 1976 thread_lock(thread); 1977 1978 /* NOTE: This mimics fill_taskthreadinfo(), which is the function used by proc_pidinfo() for 1979 * the PROC_PIDTHREADINFO flavor (which can't be used on corpses) 1980 */ 1981 retrieve_thread_basic_info(thread, &basic_info); 1982 extended_info->pth_user_time = (((uint64_t)basic_info.user_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.user_time.microseconds * NSEC_PER_USEC)); 1983 extended_info->pth_system_time = (((uint64_t)basic_info.system_time.seconds * NSEC_PER_SEC) + ((uint64_t)basic_info.system_time.microseconds * NSEC_PER_USEC)); 1984 1985 extended_info->pth_cpu_usage = basic_info.cpu_usage; 1986 extended_info->pth_policy = basic_info.policy; 1987 extended_info->pth_run_state = basic_info.run_state; 1988 extended_info->pth_flags = basic_info.flags; 1989 extended_info->pth_sleep_time = basic_info.sleep_time; 1990 extended_info->pth_curpri = thread->sched_pri; 1991 extended_info->pth_priority = thread->base_pri; 1992 extended_info->pth_maxpriority = thread->max_priority; 1993 1994 bsd_getthreadname(thread->uthread, extended_info->pth_name); 1995 1996 thread_unlock(thread); 1997 splx(s); 1998 1999 *thread_info_count = THREAD_EXTENDED_INFO_COUNT; 2000 2001 return KERN_SUCCESS; 2002 } else if (flavor == THREAD_DEBUG_INFO_INTERNAL) { 2003 #if DEVELOPMENT || DEBUG 2004 thread_debug_info_internal_t dbg_info; 2005 if (*thread_info_count < THREAD_DEBUG_INFO_INTERNAL_COUNT) { 2006 return KERN_NOT_SUPPORTED; 2007 } 2008 2009 if (thread_info_out == NULL) { 2010 return KERN_INVALID_ARGUMENT; 2011 } 2012 2013 dbg_info = __IGNORE_WCASTALIGN((thread_debug_info_internal_t)thread_info_out); 2014 dbg_info->page_creation_count = thread->t_page_creation_count; 2015 2016 *thread_info_count = THREAD_DEBUG_INFO_INTERNAL_COUNT; 2017 return KERN_SUCCESS; 2018 #endif /* DEVELOPMENT || DEBUG */ 2019 return KERN_NOT_SUPPORTED; 2020 } 2021 2022 return KERN_INVALID_ARGUMENT; 2023 } 2024 2025 void 2026 thread_read_times( 2027 thread_t thread, 2028 time_value_t *user_time, 2029 time_value_t *system_time, 2030 time_value_t *runnable_time) 2031 { 2032 clock_sec_t secs; 2033 clock_usec_t usecs; 2034 uint64_t tval_user, tval_system; 2035 2036 tval_user = timer_grab(&thread->user_timer); 2037 tval_system = timer_grab(&thread->system_timer); 2038 2039 if (thread->precise_user_kernel_time) { 2040 absolutetime_to_microtime(tval_user, &secs, &usecs); 2041 user_time->seconds = (typeof(user_time->seconds))secs; 2042 user_time->microseconds = usecs; 2043 2044 absolutetime_to_microtime(tval_system, &secs, &usecs); 2045 system_time->seconds = (typeof(system_time->seconds))secs; 2046 system_time->microseconds = usecs; 2047 } else { 2048 /* system_timer may represent either sys or user */ 2049 tval_user += tval_system; 2050 absolutetime_to_microtime(tval_user, &secs, &usecs); 2051 user_time->seconds = (typeof(user_time->seconds))secs; 2052 user_time->microseconds = usecs; 2053 2054 system_time->seconds = 0; 2055 system_time->microseconds = 0; 2056 } 2057 2058 if (runnable_time) { 2059 uint64_t tval_runnable = timer_grab(&thread->runnable_timer); 2060 absolutetime_to_microtime(tval_runnable, &secs, &usecs); 2061 runnable_time->seconds = (typeof(runnable_time->seconds))secs; 2062 runnable_time->microseconds = usecs; 2063 } 2064 } 2065 2066 uint64_t 2067 thread_get_runtime_self(void) 2068 { 2069 boolean_t interrupt_state; 2070 uint64_t runtime; 2071 thread_t thread = NULL; 2072 processor_t processor = NULL; 2073 2074 thread = current_thread(); 2075 2076 /* Not interrupt safe, as the scheduler may otherwise update timer values underneath us */ 2077 interrupt_state = ml_set_interrupts_enabled(FALSE); 2078 processor = current_processor(); 2079 timer_update(processor->thread_timer, mach_absolute_time()); 2080 runtime = (timer_grab(&thread->user_timer) + timer_grab(&thread->system_timer)); 2081 ml_set_interrupts_enabled(interrupt_state); 2082 2083 return runtime; 2084 } 2085 2086 kern_return_t 2087 thread_assign( 2088 __unused thread_t thread, 2089 __unused processor_set_t new_pset) 2090 { 2091 return KERN_FAILURE; 2092 } 2093 2094 /* 2095 * thread_assign_default: 2096 * 2097 * Special version of thread_assign for assigning threads to default 2098 * processor set. 2099 */ 2100 kern_return_t 2101 thread_assign_default( 2102 thread_t thread) 2103 { 2104 return thread_assign(thread, &pset0); 2105 } 2106 2107 /* 2108 * thread_get_assignment 2109 * 2110 * Return current assignment for this thread. 2111 */ 2112 kern_return_t 2113 thread_get_assignment( 2114 thread_t thread, 2115 processor_set_t *pset) 2116 { 2117 if (thread == NULL) { 2118 return KERN_INVALID_ARGUMENT; 2119 } 2120 2121 *pset = &pset0; 2122 2123 return KERN_SUCCESS; 2124 } 2125 2126 /* 2127 * thread_wire_internal: 2128 * 2129 * Specify that the target thread must always be able 2130 * to run and to allocate memory. 2131 */ 2132 kern_return_t 2133 thread_wire_internal( 2134 host_priv_t host_priv, 2135 thread_t thread, 2136 boolean_t wired, 2137 boolean_t *prev_state) 2138 { 2139 if (host_priv == NULL || thread != current_thread()) { 2140 return KERN_INVALID_ARGUMENT; 2141 } 2142 2143 if (prev_state) { 2144 *prev_state = (thread->options & TH_OPT_VMPRIV) != 0; 2145 } 2146 2147 if (wired) { 2148 if (!(thread->options & TH_OPT_VMPRIV)) { 2149 vm_page_free_reserve(1); /* XXX */ 2150 } 2151 thread->options |= TH_OPT_VMPRIV; 2152 } else { 2153 if (thread->options & TH_OPT_VMPRIV) { 2154 vm_page_free_reserve(-1); /* XXX */ 2155 } 2156 thread->options &= ~TH_OPT_VMPRIV; 2157 } 2158 2159 return KERN_SUCCESS; 2160 } 2161 2162 2163 /* 2164 * thread_wire: 2165 * 2166 * User-api wrapper for thread_wire_internal() 2167 */ 2168 kern_return_t 2169 thread_wire( 2170 host_priv_t host_priv, 2171 thread_t thread, 2172 boolean_t wired) 2173 { 2174 return thread_wire_internal(host_priv, thread, wired, NULL); 2175 } 2176 2177 2178 boolean_t 2179 is_vm_privileged(void) 2180 { 2181 return current_thread()->options & TH_OPT_VMPRIV ? TRUE : FALSE; 2182 } 2183 2184 boolean_t 2185 set_vm_privilege(boolean_t privileged) 2186 { 2187 boolean_t was_vmpriv; 2188 2189 if (current_thread()->options & TH_OPT_VMPRIV) { 2190 was_vmpriv = TRUE; 2191 } else { 2192 was_vmpriv = FALSE; 2193 } 2194 2195 if (privileged != FALSE) { 2196 current_thread()->options |= TH_OPT_VMPRIV; 2197 } else { 2198 current_thread()->options &= ~TH_OPT_VMPRIV; 2199 } 2200 2201 return was_vmpriv; 2202 } 2203 2204 void 2205 set_thread_rwlock_boost(void) 2206 { 2207 current_thread()->rwlock_count++; 2208 } 2209 2210 void 2211 clear_thread_rwlock_boost(void) 2212 { 2213 thread_t thread = current_thread(); 2214 2215 if ((thread->rwlock_count-- == 1) && (thread->sched_flags & TH_SFLAG_RW_PROMOTED)) { 2216 lck_rw_clear_promotion(thread, 0); 2217 } 2218 } 2219 2220 /* 2221 * XXX assuming current thread only, for now... 2222 */ 2223 void 2224 thread_guard_violation(thread_t thread, 2225 mach_exception_data_type_t code, mach_exception_data_type_t subcode, boolean_t fatal) 2226 { 2227 assert(thread == current_thread()); 2228 2229 /* Don't set up the AST for kernel threads; this check is needed to ensure 2230 * that the guard_exc_* fields in the thread structure are set only by the 2231 * current thread and therefore, don't require a lock. 2232 */ 2233 if (thread->task == kernel_task) { 2234 return; 2235 } 2236 2237 assert(EXC_GUARD_DECODE_GUARD_TYPE(code)); 2238 2239 /* 2240 * Use the saved state area of the thread structure 2241 * to store all info required to handle the AST when 2242 * returning to userspace. It's possible that there is 2243 * already a pending guard exception. If it's non-fatal, 2244 * it can only be over-written by a fatal exception code. 2245 */ 2246 if (thread->guard_exc_info.code && (thread->guard_exc_fatal || !fatal)) { 2247 return; 2248 } 2249 2250 thread->guard_exc_info.code = code; 2251 thread->guard_exc_info.subcode = subcode; 2252 thread->guard_exc_fatal = fatal ? 1 : 0; 2253 2254 spl_t s = splsched(); 2255 thread_ast_set(thread, AST_GUARD); 2256 ast_propagate(thread); 2257 splx(s); 2258 } 2259 2260 /* 2261 * guard_ast: 2262 * 2263 * Handle AST_GUARD for a thread. This routine looks at the 2264 * state saved in the thread structure to determine the cause 2265 * of this exception. Based on this value, it invokes the 2266 * appropriate routine which determines other exception related 2267 * info and raises the exception. 2268 */ 2269 void 2270 guard_ast(thread_t t) 2271 { 2272 const mach_exception_data_type_t 2273 code = t->guard_exc_info.code, 2274 subcode = t->guard_exc_info.subcode; 2275 2276 t->guard_exc_info.code = 0; 2277 t->guard_exc_info.subcode = 0; 2278 t->guard_exc_fatal = 0; 2279 2280 switch (EXC_GUARD_DECODE_GUARD_TYPE(code)) { 2281 case GUARD_TYPE_NONE: 2282 /* lingering AST_GUARD on the processor? */ 2283 break; 2284 case GUARD_TYPE_MACH_PORT: 2285 mach_port_guard_ast(t, code, subcode); 2286 break; 2287 case GUARD_TYPE_FD: 2288 fd_guard_ast(t, code, subcode); 2289 break; 2290 #if CONFIG_VNGUARD 2291 case GUARD_TYPE_VN: 2292 vn_guard_ast(t, code, subcode); 2293 break; 2294 #endif 2295 case GUARD_TYPE_VIRT_MEMORY: 2296 virt_memory_guard_ast(t, code, subcode); 2297 break; 2298 default: 2299 panic("guard_exc_info %llx %llx", code, subcode); 2300 } 2301 } 2302 2303 static void 2304 thread_cputime_callback(int warning, __unused const void *arg0, __unused const void *arg1) 2305 { 2306 if (warning == LEDGER_WARNING_ROSE_ABOVE) { 2307 #if CONFIG_TELEMETRY 2308 /* 2309 * This thread is in danger of violating the CPU usage monitor. Enable telemetry 2310 * on the entire task so there are micro-stackshots available if and when 2311 * EXC_RESOURCE is triggered. We could have chosen to enable micro-stackshots 2312 * for this thread only; but now that this task is suspect, knowing what all of 2313 * its threads are up to will be useful. 2314 */ 2315 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 1); 2316 #endif 2317 return; 2318 } 2319 2320 #if CONFIG_TELEMETRY 2321 /* 2322 * If the balance has dipped below the warning level (LEDGER_WARNING_DIPPED_BELOW) or 2323 * exceeded the limit, turn telemetry off for the task. 2324 */ 2325 telemetry_task_ctl(current_task(), TF_CPUMON_WARNING, 0); 2326 #endif 2327 2328 if (warning == 0) { 2329 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(); 2330 } 2331 } 2332 2333 void __attribute__((noinline)) 2334 SENDING_NOTIFICATION__THIS_THREAD_IS_CONSUMING_TOO_MUCH_CPU(void) 2335 { 2336 int pid = 0; 2337 task_t task = current_task(); 2338 thread_t thread = current_thread(); 2339 uint64_t tid = thread->thread_id; 2340 const char *procname = "unknown"; 2341 time_value_t thread_total_time = {0, 0}; 2342 time_value_t thread_system_time; 2343 time_value_t thread_user_time; 2344 int action; 2345 uint8_t percentage; 2346 uint32_t usage_percent = 0; 2347 uint32_t interval_sec; 2348 uint64_t interval_ns; 2349 uint64_t balance_ns; 2350 boolean_t fatal = FALSE; 2351 boolean_t send_exc_resource = TRUE; /* in addition to RESOURCE_NOTIFY */ 2352 kern_return_t kr; 2353 2354 #ifdef EXC_RESOURCE_MONITORS 2355 mach_exception_data_type_t code[EXCEPTION_CODE_MAX]; 2356 #endif /* EXC_RESOURCE_MONITORS */ 2357 struct ledger_entry_info lei; 2358 2359 assert(thread->t_threadledger != LEDGER_NULL); 2360 2361 /* 2362 * Extract the fatal bit and suspend the monitor (which clears the bit). 2363 */ 2364 task_lock(task); 2365 if (task->rusage_cpu_flags & TASK_RUSECPU_FLAGS_FATAL_CPUMON) { 2366 fatal = TRUE; 2367 send_exc_resource = TRUE; 2368 } 2369 /* Only one thread can be here at a time. Whichever makes it through 2370 * first will successfully suspend the monitor and proceed to send the 2371 * notification. Other threads will get an error trying to suspend the 2372 * monitor and give up on sending the notification. In the first release, 2373 * the monitor won't be resumed for a number of seconds, but we may 2374 * eventually need to handle low-latency resume. 2375 */ 2376 kr = task_suspend_cpumon(task); 2377 task_unlock(task); 2378 if (kr == KERN_INVALID_ARGUMENT) { 2379 return; 2380 } 2381 2382 #ifdef MACH_BSD 2383 pid = proc_selfpid(); 2384 if (task->bsd_info != NULL) { 2385 procname = proc_name_address(task->bsd_info); 2386 } 2387 #endif 2388 2389 thread_get_cpulimit(&action, &percentage, &interval_ns); 2390 2391 interval_sec = (uint32_t)(interval_ns / NSEC_PER_SEC); 2392 2393 thread_read_times(thread, &thread_user_time, &thread_system_time, NULL); 2394 time_value_add(&thread_total_time, &thread_user_time); 2395 time_value_add(&thread_total_time, &thread_system_time); 2396 ledger_get_entry_info(thread->t_threadledger, thread_ledgers.cpu_time, &lei); 2397 2398 /* credit/debit/balance/limit are in absolute time units; 2399 * the refill info is in nanoseconds. */ 2400 absolutetime_to_nanoseconds(lei.lei_balance, &balance_ns); 2401 if (lei.lei_last_refill > 0) { 2402 usage_percent = (uint32_t)((balance_ns * 100ULL) / lei.lei_last_refill); 2403 } 2404 2405 /* TODO: show task total runtime (via TASK_ABSOLUTETIME_INFO)? */ 2406 printf("process %s[%d] thread %llu caught burning CPU! It used more than %d%% CPU over %u seconds\n", 2407 procname, pid, tid, percentage, interval_sec); 2408 printf(" (actual recent usage: %d%% over ~%llu seconds)\n", 2409 usage_percent, (lei.lei_last_refill + NSEC_PER_SEC / 2) / NSEC_PER_SEC); 2410 printf(" Thread lifetime cpu usage %d.%06ds, (%d.%06d user, %d.%06d sys)\n", 2411 thread_total_time.seconds, thread_total_time.microseconds, 2412 thread_user_time.seconds, thread_user_time.microseconds, 2413 thread_system_time.seconds, thread_system_time.microseconds); 2414 printf(" Ledger balance: %lld; mabs credit: %lld; mabs debit: %lld\n", 2415 lei.lei_balance, lei.lei_credit, lei.lei_debit); 2416 printf(" mabs limit: %llu; mabs period: %llu ns; last refill: %llu ns%s.\n", 2417 lei.lei_limit, lei.lei_refill_period, lei.lei_last_refill, 2418 (fatal ? " [fatal violation]" : "")); 2419 2420 /* 2421 * For now, send RESOURCE_NOTIFY in parallel with EXC_RESOURCE. Once 2422 * we have logging parity, we will stop sending EXC_RESOURCE (24508922). 2423 */ 2424 2425 /* RESOURCE_NOTIFY MIG specifies nanoseconds of CPU time */ 2426 lei.lei_balance = balance_ns; 2427 absolutetime_to_nanoseconds(lei.lei_limit, &lei.lei_limit); 2428 trace_resource_violation(RMON_CPUUSAGE_VIOLATED, &lei); 2429 kr = send_resource_violation(send_cpu_usage_violation, task, &lei, 2430 fatal ? kRNFatalLimitFlag : 0); 2431 if (kr) { 2432 printf("send_resource_violation(CPU usage, ...): error %#x\n", kr); 2433 } 2434 2435 #ifdef EXC_RESOURCE_MONITORS 2436 if (send_exc_resource) { 2437 if (disable_exc_resource) { 2438 printf("process %s[%d] thread %llu caught burning CPU! " 2439 "EXC_RESOURCE%s supressed by a boot-arg\n", 2440 procname, pid, tid, fatal ? " (and termination)" : ""); 2441 return; 2442 } 2443 2444 if (audio_active) { 2445 printf("process %s[%d] thread %llu caught burning CPU! " 2446 "EXC_RESOURCE & termination supressed due to audio playback\n", 2447 procname, pid, tid); 2448 return; 2449 } 2450 } 2451 2452 2453 if (send_exc_resource) { 2454 code[0] = code[1] = 0; 2455 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_CPU); 2456 if (fatal) { 2457 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR_FATAL); 2458 } else { 2459 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_CPU_MONITOR); 2460 } 2461 EXC_RESOURCE_CPUMONITOR_ENCODE_INTERVAL(code[0], interval_sec); 2462 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[0], percentage); 2463 EXC_RESOURCE_CPUMONITOR_ENCODE_PERCENTAGE(code[1], usage_percent); 2464 exception_triage(EXC_RESOURCE, code, EXCEPTION_CODE_MAX); 2465 } 2466 #endif /* EXC_RESOURCE_MONITORS */ 2467 2468 if (fatal) { 2469 #if CONFIG_JETSAM 2470 jetsam_on_ledger_cpulimit_exceeded(); 2471 #else 2472 task_terminate_internal(task); 2473 #endif 2474 } 2475 } 2476 2477 #if DEVELOPMENT || DEBUG 2478 void __attribute__((noinline)) 2479 SENDING_NOTIFICATION__TASK_HAS_TOO_MANY_THREADS(task_t task, int thread_count) 2480 { 2481 mach_exception_data_type_t code[EXCEPTION_CODE_MAX] = {0}; 2482 int pid = task_pid(task); 2483 char procname[MAXCOMLEN + 1] = "unknown"; 2484 2485 if (pid == 1) { 2486 /* 2487 * Cannot suspend launchd 2488 */ 2489 return; 2490 } 2491 2492 proc_name(pid, procname, sizeof(procname)); 2493 2494 if (disable_exc_resource) { 2495 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE " 2496 "supressed by a boot-arg. \n", procname, pid, thread_count); 2497 return; 2498 } 2499 2500 if (audio_active) { 2501 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE " 2502 "supressed due to audio playback.\n", procname, pid, thread_count); 2503 return; 2504 } 2505 2506 if (exc_via_corpse_forking == 0) { 2507 printf("process %s[%d] crossed thread count high watermark (%d), EXC_RESOURCE " 2508 "supressed due to corpse forking being disabled.\n", procname, pid, 2509 thread_count); 2510 return; 2511 } 2512 2513 printf("process %s[%d] crossed thread count high watermark (%d), sending " 2514 "EXC_RESOURCE\n", procname, pid, thread_count); 2515 2516 EXC_RESOURCE_ENCODE_TYPE(code[0], RESOURCE_TYPE_THREADS); 2517 EXC_RESOURCE_ENCODE_FLAVOR(code[0], FLAVOR_THREADS_HIGH_WATERMARK); 2518 EXC_RESOURCE_THREADS_ENCODE_THREADS(code[0], thread_count); 2519 2520 task_enqueue_exception_with_corpse(task, EXC_RESOURCE, code, EXCEPTION_CODE_MAX, NULL); 2521 } 2522 #endif /* DEVELOPMENT || DEBUG */ 2523 2524 void 2525 thread_update_io_stats(thread_t thread, int size, int io_flags) 2526 { 2527 int io_tier; 2528 2529 if (thread->thread_io_stats == NULL || thread->task->task_io_stats == NULL) { 2530 return; 2531 } 2532 2533 if (io_flags & DKIO_READ) { 2534 UPDATE_IO_STATS(thread->thread_io_stats->disk_reads, size); 2535 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->disk_reads, size); 2536 } 2537 2538 if (io_flags & DKIO_META) { 2539 UPDATE_IO_STATS(thread->thread_io_stats->metadata, size); 2540 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->metadata, size); 2541 } 2542 2543 if (io_flags & DKIO_PAGING) { 2544 UPDATE_IO_STATS(thread->thread_io_stats->paging, size); 2545 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->paging, size); 2546 } 2547 2548 io_tier = ((io_flags & DKIO_TIER_MASK) >> DKIO_TIER_SHIFT); 2549 assert(io_tier < IO_NUM_PRIORITIES); 2550 2551 UPDATE_IO_STATS(thread->thread_io_stats->io_priority[io_tier], size); 2552 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->io_priority[io_tier], size); 2553 2554 /* Update Total I/O Counts */ 2555 UPDATE_IO_STATS(thread->thread_io_stats->total_io, size); 2556 UPDATE_IO_STATS_ATOMIC(thread->task->task_io_stats->total_io, size); 2557 2558 if (!(io_flags & DKIO_READ)) { 2559 DTRACE_IO3(physical_writes, struct task *, thread->task, uint32_t, size, int, io_flags); 2560 ledger_credit(thread->task->ledger, task_ledgers.physical_writes, size); 2561 } 2562 } 2563 2564 static void 2565 init_thread_ledgers(void) 2566 { 2567 ledger_template_t t; 2568 int idx; 2569 2570 assert(thread_ledger_template == NULL); 2571 2572 if ((t = ledger_template_create("Per-thread ledger")) == NULL) { 2573 panic("couldn't create thread ledger template"); 2574 } 2575 2576 if ((idx = ledger_entry_add(t, "cpu_time", "sched", "ns")) < 0) { 2577 panic("couldn't create cpu_time entry for thread ledger template"); 2578 } 2579 2580 if (ledger_set_callback(t, idx, thread_cputime_callback, NULL, NULL) < 0) { 2581 panic("couldn't set thread ledger callback for cpu_time entry"); 2582 } 2583 2584 thread_ledgers.cpu_time = idx; 2585 2586 ledger_template_complete(t); 2587 thread_ledger_template = t; 2588 } 2589 2590 /* 2591 * Returns currently applied CPU usage limit, or 0/0 if none is applied. 2592 */ 2593 int 2594 thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns) 2595 { 2596 int64_t abstime = 0; 2597 uint64_t limittime = 0; 2598 thread_t thread = current_thread(); 2599 2600 *percentage = 0; 2601 *interval_ns = 0; 2602 *action = 0; 2603 2604 if (thread->t_threadledger == LEDGER_NULL) { 2605 /* 2606 * This thread has no per-thread ledger, so it can't possibly 2607 * have a CPU limit applied. 2608 */ 2609 return KERN_SUCCESS; 2610 } 2611 2612 ledger_get_period(thread->t_threadledger, thread_ledgers.cpu_time, interval_ns); 2613 ledger_get_limit(thread->t_threadledger, thread_ledgers.cpu_time, &abstime); 2614 2615 if ((abstime == LEDGER_LIMIT_INFINITY) || (*interval_ns == 0)) { 2616 /* 2617 * This thread's CPU time ledger has no period or limit; so it 2618 * doesn't have a CPU limit applied. 2619 */ 2620 return KERN_SUCCESS; 2621 } 2622 2623 /* 2624 * This calculation is the converse to the one in thread_set_cpulimit(). 2625 */ 2626 absolutetime_to_nanoseconds(abstime, &limittime); 2627 *percentage = (uint8_t)((limittime * 100ULL) / *interval_ns); 2628 assert(*percentage <= 100); 2629 2630 if (thread->options & TH_OPT_PROC_CPULIMIT) { 2631 assert((thread->options & TH_OPT_PRVT_CPULIMIT) == 0); 2632 2633 *action = THREAD_CPULIMIT_BLOCK; 2634 } else if (thread->options & TH_OPT_PRVT_CPULIMIT) { 2635 assert((thread->options & TH_OPT_PROC_CPULIMIT) == 0); 2636 2637 *action = THREAD_CPULIMIT_EXCEPTION; 2638 } else { 2639 *action = THREAD_CPULIMIT_DISABLE; 2640 } 2641 2642 return KERN_SUCCESS; 2643 } 2644 2645 /* 2646 * Set CPU usage limit on a thread. 2647 * 2648 * Calling with percentage of 0 will unset the limit for this thread. 2649 */ 2650 int 2651 thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns) 2652 { 2653 thread_t thread = current_thread(); 2654 ledger_t l; 2655 uint64_t limittime = 0; 2656 uint64_t abstime = 0; 2657 2658 assert(percentage <= 100); 2659 2660 if (action == THREAD_CPULIMIT_DISABLE) { 2661 /* 2662 * Remove CPU limit, if any exists. 2663 */ 2664 if (thread->t_threadledger != LEDGER_NULL) { 2665 l = thread->t_threadledger; 2666 ledger_set_limit(l, thread_ledgers.cpu_time, LEDGER_LIMIT_INFINITY, 0); 2667 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_IGNORE); 2668 thread->options &= ~(TH_OPT_PROC_CPULIMIT | TH_OPT_PRVT_CPULIMIT); 2669 } 2670 2671 return 0; 2672 } 2673 2674 if (interval_ns < MINIMUM_CPULIMIT_INTERVAL_MS * NSEC_PER_MSEC) { 2675 return KERN_INVALID_ARGUMENT; 2676 } 2677 2678 l = thread->t_threadledger; 2679 if (l == LEDGER_NULL) { 2680 /* 2681 * This thread doesn't yet have a per-thread ledger; so create one with the CPU time entry active. 2682 */ 2683 if ((l = ledger_instantiate(thread_ledger_template, LEDGER_CREATE_INACTIVE_ENTRIES)) == LEDGER_NULL) { 2684 return KERN_RESOURCE_SHORTAGE; 2685 } 2686 2687 /* 2688 * We are the first to create this thread's ledger, so only activate our entry. 2689 */ 2690 ledger_entry_setactive(l, thread_ledgers.cpu_time); 2691 thread->t_threadledger = l; 2692 } 2693 2694 /* 2695 * The limit is specified as a percentage of CPU over an interval in nanoseconds. 2696 * Calculate the amount of CPU time that the thread needs to consume in order to hit the limit. 2697 */ 2698 limittime = (interval_ns * percentage) / 100; 2699 nanoseconds_to_absolutetime(limittime, &abstime); 2700 ledger_set_limit(l, thread_ledgers.cpu_time, abstime, cpumon_ustackshots_trigger_pct); 2701 /* 2702 * Refill the thread's allotted CPU time every interval_ns nanoseconds. 2703 */ 2704 ledger_set_period(l, thread_ledgers.cpu_time, interval_ns); 2705 2706 if (action == THREAD_CPULIMIT_EXCEPTION) { 2707 /* 2708 * We don't support programming the CPU usage monitor on a task if any of its 2709 * threads have a per-thread blocking CPU limit configured. 2710 */ 2711 if (thread->options & TH_OPT_PRVT_CPULIMIT) { 2712 panic("CPU usage monitor activated, but blocking thread limit exists"); 2713 } 2714 2715 /* 2716 * Make a note that this thread's CPU limit is being used for the task-wide CPU 2717 * usage monitor. We don't have to arm the callback which will trigger the 2718 * exception, because that was done for us in ledger_instantiate (because the 2719 * ledger template used has a default callback). 2720 */ 2721 thread->options |= TH_OPT_PROC_CPULIMIT; 2722 } else { 2723 /* 2724 * We deliberately override any CPU limit imposed by a task-wide limit (eg 2725 * CPU usage monitor). 2726 */ 2727 thread->options &= ~TH_OPT_PROC_CPULIMIT; 2728 2729 thread->options |= TH_OPT_PRVT_CPULIMIT; 2730 /* The per-thread ledger template by default has a callback for CPU time */ 2731 ledger_disable_callback(l, thread_ledgers.cpu_time); 2732 ledger_set_action(l, thread_ledgers.cpu_time, LEDGER_ACTION_BLOCK); 2733 } 2734 2735 return 0; 2736 } 2737 2738 void 2739 thread_sched_call( 2740 thread_t thread, 2741 sched_call_t call) 2742 { 2743 assert((thread->state & TH_WAIT_REPORT) == 0); 2744 thread->sched_call = call; 2745 } 2746 2747 uint64_t 2748 thread_tid( 2749 thread_t thread) 2750 { 2751 return thread != THREAD_NULL? thread->thread_id: 0; 2752 } 2753 2754 uint16_t 2755 thread_set_tag(thread_t th, uint16_t tag) 2756 { 2757 return thread_set_tag_internal(th, tag); 2758 } 2759 2760 uint16_t 2761 thread_get_tag(thread_t th) 2762 { 2763 return thread_get_tag_internal(th); 2764 } 2765 2766 uint64_t 2767 thread_last_run_time(thread_t th) 2768 { 2769 return th->last_run_time; 2770 } 2771 2772 uint64_t 2773 thread_dispatchqaddr( 2774 thread_t thread) 2775 { 2776 uint64_t dispatchqueue_addr; 2777 uint64_t thread_handle; 2778 2779 if (thread == THREAD_NULL) { 2780 return 0; 2781 } 2782 2783 thread_handle = thread->machine.cthread_self; 2784 if (thread_handle == 0) { 2785 return 0; 2786 } 2787 2788 if (thread->inspection == TRUE) { 2789 dispatchqueue_addr = thread_handle + get_task_dispatchqueue_offset(thread->task); 2790 } else if (thread->task->bsd_info) { 2791 dispatchqueue_addr = thread_handle + get_dispatchqueue_offset_from_proc(thread->task->bsd_info); 2792 } else { 2793 dispatchqueue_addr = 0; 2794 } 2795 2796 return dispatchqueue_addr; 2797 } 2798 2799 uint64_t 2800 thread_rettokern_addr( 2801 thread_t thread) 2802 { 2803 uint64_t rettokern_addr; 2804 uint64_t rettokern_offset; 2805 uint64_t thread_handle; 2806 2807 if (thread == THREAD_NULL) { 2808 return 0; 2809 } 2810 2811 thread_handle = thread->machine.cthread_self; 2812 if (thread_handle == 0) { 2813 return 0; 2814 } 2815 2816 if (thread->task->bsd_info) { 2817 rettokern_offset = get_return_to_kernel_offset_from_proc(thread->task->bsd_info); 2818 2819 /* Return 0 if return to kernel offset is not initialized. */ 2820 if (rettokern_offset == 0) { 2821 rettokern_addr = 0; 2822 } else { 2823 rettokern_addr = thread_handle + rettokern_offset; 2824 } 2825 } else { 2826 rettokern_addr = 0; 2827 } 2828 2829 return rettokern_addr; 2830 } 2831 2832 /* 2833 * Export routines to other components for things that are done as macros 2834 * within the osfmk component. 2835 */ 2836 2837 #undef thread_mtx_lock 2838 void thread_mtx_lock(thread_t thread); 2839 void 2840 thread_mtx_lock(thread_t thread) 2841 { 2842 lck_mtx_lock(&thread->mutex); 2843 } 2844 2845 #undef thread_mtx_unlock 2846 void thread_mtx_unlock(thread_t thread); 2847 void 2848 thread_mtx_unlock(thread_t thread) 2849 { 2850 lck_mtx_unlock(&thread->mutex); 2851 } 2852 2853 #undef thread_reference 2854 void thread_reference(thread_t thread); 2855 void 2856 thread_reference( 2857 thread_t thread) 2858 { 2859 if (thread != THREAD_NULL) { 2860 thread_reference_internal(thread); 2861 } 2862 } 2863 2864 #undef thread_should_halt 2865 2866 boolean_t 2867 thread_should_halt( 2868 thread_t th) 2869 { 2870 return thread_should_halt_fast(th); 2871 } 2872 2873 /* 2874 * thread_set_voucher_name - reset the voucher port name bound to this thread 2875 * 2876 * Conditions: nothing locked 2877 */ 2878 2879 kern_return_t 2880 thread_set_voucher_name(mach_port_name_t voucher_name) 2881 { 2882 thread_t thread = current_thread(); 2883 ipc_voucher_t new_voucher = IPC_VOUCHER_NULL; 2884 ipc_voucher_t voucher; 2885 ledger_t bankledger = NULL; 2886 struct thread_group *banktg = NULL; 2887 uint32_t persona_id = 0; 2888 2889 if (MACH_PORT_DEAD == voucher_name) { 2890 return KERN_INVALID_RIGHT; 2891 } 2892 2893 /* 2894 * agressively convert to voucher reference 2895 */ 2896 if (MACH_PORT_VALID(voucher_name)) { 2897 new_voucher = convert_port_name_to_voucher(voucher_name); 2898 if (IPC_VOUCHER_NULL == new_voucher) { 2899 return KERN_INVALID_ARGUMENT; 2900 } 2901 } 2902 bank_get_bank_ledger_thread_group_and_persona(new_voucher, &bankledger, &banktg, &persona_id); 2903 2904 thread_mtx_lock(thread); 2905 voucher = thread->ith_voucher; 2906 thread->ith_voucher_name = voucher_name; 2907 thread->ith_voucher = new_voucher; 2908 thread_mtx_unlock(thread); 2909 2910 bank_swap_thread_bank_ledger(thread, bankledger); 2911 #if CONFIG_THREAD_GROUPS 2912 thread_group_set_bank(thread, banktg); 2913 #endif /* CONFIG_THREAD_GROUPS */ 2914 2915 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2916 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 2917 (uintptr_t)thread_tid(thread), 2918 (uintptr_t)voucher_name, 2919 VM_KERNEL_ADDRPERM((uintptr_t)new_voucher), 2920 persona_id, 0); 2921 2922 if (IPC_VOUCHER_NULL != voucher) { 2923 ipc_voucher_release(voucher); 2924 } 2925 2926 return KERN_SUCCESS; 2927 } 2928 2929 /* 2930 * thread_get_mach_voucher - return a voucher reference for the specified thread voucher 2931 * 2932 * Conditions: nothing locked 2933 * 2934 * NOTE: At the moment, there is no distinction between the current and effective 2935 * vouchers because we only set them at the thread level currently. 2936 */ 2937 kern_return_t 2938 thread_get_mach_voucher( 2939 thread_act_t thread, 2940 mach_voucher_selector_t __unused which, 2941 ipc_voucher_t *voucherp) 2942 { 2943 ipc_voucher_t voucher; 2944 2945 if (THREAD_NULL == thread) { 2946 return KERN_INVALID_ARGUMENT; 2947 } 2948 2949 thread_mtx_lock(thread); 2950 voucher = thread->ith_voucher; 2951 2952 if (IPC_VOUCHER_NULL != voucher) { 2953 ipc_voucher_reference(voucher); 2954 thread_mtx_unlock(thread); 2955 *voucherp = voucher; 2956 return KERN_SUCCESS; 2957 } 2958 2959 thread_mtx_unlock(thread); 2960 2961 *voucherp = IPC_VOUCHER_NULL; 2962 return KERN_SUCCESS; 2963 } 2964 2965 /* 2966 * thread_set_mach_voucher - set a voucher reference for the specified thread voucher 2967 * 2968 * Conditions: callers holds a reference on the voucher. 2969 * nothing locked. 2970 * 2971 * We grab another reference to the voucher and bind it to the thread. 2972 * The old voucher reference associated with the thread is 2973 * discarded. 2974 */ 2975 kern_return_t 2976 thread_set_mach_voucher( 2977 thread_t thread, 2978 ipc_voucher_t voucher) 2979 { 2980 ipc_voucher_t old_voucher; 2981 ledger_t bankledger = NULL; 2982 struct thread_group *banktg = NULL; 2983 uint32_t persona_id = 0; 2984 2985 if (THREAD_NULL == thread) { 2986 return KERN_INVALID_ARGUMENT; 2987 } 2988 2989 bank_get_bank_ledger_thread_group_and_persona(voucher, &bankledger, &banktg, &persona_id); 2990 2991 thread_mtx_lock(thread); 2992 /* 2993 * Once the thread is started, we will look at `ith_voucher` without 2994 * holding any lock. 2995 * 2996 * Setting the voucher hence can only be done by current_thread() or 2997 * before it started. "started" flips under the thread mutex and must be 2998 * tested under it too. 2999 */ 3000 if (thread != current_thread() && thread->started) { 3001 thread_mtx_unlock(thread); 3002 return KERN_INVALID_ARGUMENT; 3003 } 3004 3005 ipc_voucher_reference(voucher); 3006 old_voucher = thread->ith_voucher; 3007 thread->ith_voucher = voucher; 3008 thread->ith_voucher_name = MACH_PORT_NULL; 3009 thread_mtx_unlock(thread); 3010 3011 bank_swap_thread_bank_ledger(thread, bankledger); 3012 #if CONFIG_THREAD_GROUPS 3013 thread_group_set_bank(thread, banktg); 3014 #endif /* CONFIG_THREAD_GROUPS */ 3015 3016 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 3017 MACHDBG_CODE(DBG_MACH_IPC, MACH_THREAD_SET_VOUCHER) | DBG_FUNC_NONE, 3018 (uintptr_t)thread_tid(thread), 3019 (uintptr_t)MACH_PORT_NULL, 3020 VM_KERNEL_ADDRPERM((uintptr_t)voucher), 3021 persona_id, 0); 3022 3023 ipc_voucher_release(old_voucher); 3024 3025 return KERN_SUCCESS; 3026 } 3027 3028 /* 3029 * thread_swap_mach_voucher - swap a voucher reference for the specified thread voucher 3030 * 3031 * Conditions: callers holds a reference on the new and presumed old voucher(s). 3032 * nothing locked. 3033 * 3034 * This function is no longer supported. 3035 */ 3036 kern_return_t 3037 thread_swap_mach_voucher( 3038 __unused thread_t thread, 3039 __unused ipc_voucher_t new_voucher, 3040 ipc_voucher_t *in_out_old_voucher) 3041 { 3042 /* 3043 * Currently this function is only called from a MIG generated 3044 * routine which doesn't release the reference on the voucher 3045 * addressed by in_out_old_voucher. To avoid leaking this reference, 3046 * a call to release it has been added here. 3047 */ 3048 ipc_voucher_release(*in_out_old_voucher); 3049 return KERN_NOT_SUPPORTED; 3050 } 3051 3052 /* 3053 * thread_get_current_voucher_origin_pid - get the pid of the originator of the current voucher. 3054 */ 3055 kern_return_t 3056 thread_get_current_voucher_origin_pid( 3057 int32_t *pid) 3058 { 3059 uint32_t buf_size; 3060 kern_return_t kr; 3061 thread_t thread = current_thread(); 3062 3063 buf_size = sizeof(*pid); 3064 kr = mach_voucher_attr_command(thread->ith_voucher, 3065 MACH_VOUCHER_ATTR_KEY_BANK, 3066 BANK_ORIGINATOR_PID, 3067 NULL, 3068 0, 3069 (mach_voucher_attr_content_t)pid, 3070 &buf_size); 3071 3072 return kr; 3073 } 3074 3075 #if CONFIG_THREAD_GROUPS 3076 /* 3077 * Returns the current thread's voucher-carried thread group 3078 * 3079 * Reference is borrowed from this being the current voucher, so it does NOT 3080 * return a reference to the group. 3081 */ 3082 struct thread_group * 3083 thread_get_current_voucher_thread_group(thread_t thread) 3084 { 3085 assert(thread == current_thread()); 3086 3087 if (thread->ith_voucher == NULL) { 3088 return NULL; 3089 } 3090 3091 ledger_t bankledger = NULL; 3092 struct thread_group *banktg = NULL; 3093 3094 bank_get_bank_ledger_thread_group_and_persona(thread->ith_voucher, &bankledger, &banktg, NULL); 3095 3096 return banktg; 3097 } 3098 3099 #endif /* CONFIG_THREAD_GROUPS */ 3100 3101 boolean_t 3102 thread_has_thread_name(thread_t th) 3103 { 3104 if ((th) && (th->uthread)) { 3105 return bsd_hasthreadname(th->uthread); 3106 } 3107 3108 /* 3109 * This is an odd case; clients may set the thread name based on the lack of 3110 * a name, but in this context there is no uthread to attach the name to. 3111 */ 3112 return FALSE; 3113 } 3114 3115 void 3116 thread_set_thread_name(thread_t th, const char* name) 3117 { 3118 if ((th) && (th->uthread) && name) { 3119 bsd_setthreadname(th->uthread, name); 3120 } 3121 } 3122 3123 void 3124 thread_get_thread_name(thread_t th, char* name) 3125 { 3126 if (!name) { 3127 return; 3128 } 3129 if ((th) && (th->uthread)) { 3130 bsd_getthreadname(th->uthread, name); 3131 } else { 3132 name[0] = '\0'; 3133 } 3134 } 3135 3136 void 3137 thread_set_honor_qlimit(thread_t thread) 3138 { 3139 thread->options |= TH_OPT_HONOR_QLIMIT; 3140 } 3141 3142 void 3143 thread_clear_honor_qlimit(thread_t thread) 3144 { 3145 thread->options &= (~TH_OPT_HONOR_QLIMIT); 3146 } 3147 3148 /* 3149 * thread_enable_send_importance - set/clear the SEND_IMPORTANCE thread option bit. 3150 */ 3151 void 3152 thread_enable_send_importance(thread_t thread, boolean_t enable) 3153 { 3154 if (enable == TRUE) { 3155 thread->options |= TH_OPT_SEND_IMPORTANCE; 3156 } else { 3157 thread->options &= ~TH_OPT_SEND_IMPORTANCE; 3158 } 3159 } 3160 3161 /* 3162 * thread_set_allocation_name - . 3163 */ 3164 3165 kern_allocation_name_t 3166 thread_set_allocation_name(kern_allocation_name_t new_name) 3167 { 3168 kern_allocation_name_t ret; 3169 thread_kernel_state_t kstate = thread_get_kernel_state(current_thread()); 3170 ret = kstate->allocation_name; 3171 // fifo 3172 if (!new_name || !kstate->allocation_name) { 3173 kstate->allocation_name = new_name; 3174 } 3175 return ret; 3176 } 3177 3178 void * 3179 thread_iokit_tls_get(uint32_t index) 3180 { 3181 assert(index < THREAD_SAVE_IOKIT_TLS_COUNT); 3182 return current_thread()->saved.iokit.tls[index]; 3183 } 3184 3185 void 3186 thread_iokit_tls_set(uint32_t index, void * data) 3187 { 3188 assert(index < THREAD_SAVE_IOKIT_TLS_COUNT); 3189 current_thread()->saved.iokit.tls[index] = data; 3190 } 3191 3192 uint64_t 3193 thread_get_last_wait_duration(thread_t thread) 3194 { 3195 return thread->last_made_runnable_time - thread->last_run_time; 3196 } 3197 3198 integer_t 3199 thread_kern_get_pri(thread_t thr) 3200 { 3201 return thr->base_pri; 3202 } 3203 3204 void 3205 thread_kern_set_pri(thread_t thr, integer_t pri) 3206 { 3207 sched_set_kernel_thread_priority(thr, pri); 3208 } 3209 3210 integer_t 3211 thread_kern_get_kernel_maxpri(void) 3212 { 3213 return MAXPRI_KERNEL; 3214 } 3215 /* 3216 * thread_port_with_flavor_notify 3217 * 3218 * Called whenever the Mach port system detects no-senders on 3219 * the thread inspect or read port. These ports are allocated lazily and 3220 * should be deallocated here when there are no senders remaining. 3221 */ 3222 void 3223 thread_port_with_flavor_notify(mach_msg_header_t *msg) 3224 { 3225 mach_no_senders_notification_t *notification = (void *)msg; 3226 ipc_port_t port = notification->not_header.msgh_remote_port; 3227 thread_t thread; 3228 mach_thread_flavor_t flavor; 3229 ipc_kobject_type_t kotype; 3230 3231 ip_lock(port); 3232 if (port->ip_srights > 0) { 3233 ip_unlock(port); 3234 return; 3235 } 3236 thread = (thread_t)ipc_kobject_get(port); 3237 kotype = ip_kotype(port); 3238 if (thread != THREAD_NULL) { 3239 assert((IKOT_THREAD_READ == kotype) || (IKOT_THREAD_INSPECT == kotype)); 3240 thread_reference_internal(thread); 3241 } 3242 ip_unlock(port); 3243 3244 if (thread == THREAD_NULL) { 3245 /* The thread is exiting or disabled; it will eventually deallocate the port */ 3246 return; 3247 } 3248 3249 if (kotype == IKOT_THREAD_READ) { 3250 flavor = THREAD_FLAVOR_READ; 3251 } else { 3252 flavor = THREAD_FLAVOR_INSPECT; 3253 } 3254 3255 thread_mtx_lock(thread); 3256 ip_lock(port); 3257 /* 3258 * If the port is no longer active, then ipc_thread_terminate() ran 3259 * and destroyed the kobject already. Just deallocate the task 3260 * ref we took and go away. 3261 * 3262 * It is also possible that several nsrequests are in flight, 3263 * only one shall NULL-out the port entry, and this is the one 3264 * that gets to dealloc the port. 3265 * 3266 * Check for a stale no-senders notification. A call to any function 3267 * that vends out send rights to this port could resurrect it between 3268 * this notification being generated and actually being handled here. 3269 */ 3270 if (!ip_active(port) || 3271 thread->ith_thread_ports[flavor] != port || 3272 port->ip_srights > 0) { 3273 ip_unlock(port); 3274 thread_mtx_unlock(thread); 3275 thread_deallocate(thread); 3276 return; 3277 } 3278 3279 assert(thread->ith_thread_ports[flavor] == port); 3280 thread->ith_thread_ports[flavor] = IP_NULL; 3281 ipc_kobject_set_atomically(port, IKO_NULL, IKOT_NONE); 3282 ip_unlock(port); 3283 thread_mtx_unlock(thread); 3284 thread_deallocate(thread); 3285 3286 ipc_port_dealloc_kernel(port); 3287 } 3288 3289 /* 3290 * The 'thread_region_page_shift' is used by footprint 3291 * to specify the page size that it will use to 3292 * accomplish its accounting work on the task being 3293 * inspected. Since footprint uses a thread for each 3294 * task that it works on, we need to keep the page_shift 3295 * on a per-thread basis. 3296 */ 3297 3298 int 3299 thread_self_region_page_shift(void) 3300 { 3301 /* 3302 * Return the page shift that this thread 3303 * would like to use for its accounting work. 3304 */ 3305 return current_thread()->thread_region_page_shift; 3306 } 3307 3308 void 3309 thread_self_region_page_shift_set( 3310 int pgshift) 3311 { 3312 /* 3313 * Set the page shift that this thread 3314 * would like to use for its accounting work 3315 * when dealing with a task. 3316 */ 3317 current_thread()->thread_region_page_shift = pgshift; 3318 } 3319 3320 #if CONFIG_DTRACE 3321 uint32_t 3322 dtrace_get_thread_predcache(thread_t thread) 3323 { 3324 if (thread != THREAD_NULL) { 3325 return thread->t_dtrace_predcache; 3326 } else { 3327 return 0; 3328 } 3329 } 3330 3331 int64_t 3332 dtrace_get_thread_vtime(thread_t thread) 3333 { 3334 if (thread != THREAD_NULL) { 3335 return thread->t_dtrace_vtime; 3336 } else { 3337 return 0; 3338 } 3339 } 3340 3341 int 3342 dtrace_get_thread_last_cpu_id(thread_t thread) 3343 { 3344 if ((thread != THREAD_NULL) && (thread->last_processor != PROCESSOR_NULL)) { 3345 return thread->last_processor->cpu_id; 3346 } else { 3347 return -1; 3348 } 3349 } 3350 3351 int64_t 3352 dtrace_get_thread_tracing(thread_t thread) 3353 { 3354 if (thread != THREAD_NULL) { 3355 return thread->t_dtrace_tracing; 3356 } else { 3357 return 0; 3358 } 3359 } 3360 3361 uint16_t 3362 dtrace_get_thread_inprobe(thread_t thread) 3363 { 3364 if (thread != THREAD_NULL) { 3365 return thread->t_dtrace_inprobe; 3366 } else { 3367 return 0; 3368 } 3369 } 3370 3371 vm_offset_t 3372 dtrace_get_kernel_stack(thread_t thread) 3373 { 3374 if (thread != THREAD_NULL) { 3375 return thread->kernel_stack; 3376 } else { 3377 return 0; 3378 } 3379 } 3380 3381 #if KASAN 3382 struct kasan_thread_data * 3383 kasan_get_thread_data(thread_t thread) 3384 { 3385 return &thread->kasan_data; 3386 } 3387 #endif 3388 3389 #if CONFIG_KSANCOV 3390 void ** 3391 __sanitizer_get_thread_data(thread_t thread) 3392 { 3393 return &thread->ksancov_data; 3394 } 3395 #endif 3396 3397 int64_t 3398 dtrace_calc_thread_recent_vtime(thread_t thread) 3399 { 3400 if (thread != THREAD_NULL) { 3401 processor_t processor = current_processor(); 3402 uint64_t abstime = mach_absolute_time(); 3403 timer_t timer; 3404 3405 timer = processor->thread_timer; 3406 3407 return timer_grab(&(thread->system_timer)) + timer_grab(&(thread->user_timer)) + 3408 (abstime - timer->tstamp); /* XXX need interrupts off to prevent missed time? */ 3409 } else { 3410 return 0; 3411 } 3412 } 3413 3414 void 3415 dtrace_set_thread_predcache(thread_t thread, uint32_t predcache) 3416 { 3417 if (thread != THREAD_NULL) { 3418 thread->t_dtrace_predcache = predcache; 3419 } 3420 } 3421 3422 void 3423 dtrace_set_thread_vtime(thread_t thread, int64_t vtime) 3424 { 3425 if (thread != THREAD_NULL) { 3426 thread->t_dtrace_vtime = vtime; 3427 } 3428 } 3429 3430 void 3431 dtrace_set_thread_tracing(thread_t thread, int64_t accum) 3432 { 3433 if (thread != THREAD_NULL) { 3434 thread->t_dtrace_tracing = accum; 3435 } 3436 } 3437 3438 void 3439 dtrace_set_thread_inprobe(thread_t thread, uint16_t inprobe) 3440 { 3441 if (thread != THREAD_NULL) { 3442 thread->t_dtrace_inprobe = inprobe; 3443 } 3444 } 3445 3446 vm_offset_t 3447 dtrace_set_thread_recover(thread_t thread, vm_offset_t recover) 3448 { 3449 vm_offset_t prev = 0; 3450 3451 if (thread != THREAD_NULL) { 3452 prev = thread->recover; 3453 thread->recover = recover; 3454 } 3455 return prev; 3456 } 3457 3458 vm_offset_t 3459 dtrace_sign_and_set_thread_recover(thread_t thread, vm_offset_t recover) 3460 { 3461 #if defined(HAS_APPLE_PAC) 3462 return dtrace_set_thread_recover(thread, 3463 (vm_address_t)ptrauth_sign_unauthenticated((void *)recover, 3464 ptrauth_key_function_pointer, 3465 ptrauth_blend_discriminator(&thread->recover, PAC_DISCRIMINATOR_RECOVER))); 3466 #else /* defined(HAS_APPLE_PAC) */ 3467 return dtrace_set_thread_recover(thread, recover); 3468 #endif /* defined(HAS_APPLE_PAC) */ 3469 } 3470 3471 void 3472 dtrace_thread_bootstrap(void) 3473 { 3474 task_t task = current_task(); 3475 3476 if (task->thread_count == 1) { 3477 thread_t thread = current_thread(); 3478 if (thread->t_dtrace_flags & TH_DTRACE_EXECSUCCESS) { 3479 thread->t_dtrace_flags &= ~TH_DTRACE_EXECSUCCESS; 3480 DTRACE_PROC(exec__success); 3481 KDBG(BSDDBG_CODE(DBG_BSD_PROC, BSD_PROC_EXEC), 3482 task_pid(task)); 3483 } 3484 DTRACE_PROC(start); 3485 } 3486 DTRACE_PROC(lwp__start); 3487 } 3488 3489 void 3490 dtrace_thread_didexec(thread_t thread) 3491 { 3492 thread->t_dtrace_flags |= TH_DTRACE_EXECSUCCESS; 3493 } 3494 #endif /* CONFIG_DTRACE */