1  /*
  2   * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
  3   *
  4   * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  5   *
  6   * This file contains Original Code and/or Modifications of Original Code
  7   * as defined in and that are subject to the Apple Public Source License
  8   * Version 2.0 (the 'License'). You may not use this file except in
  9   * compliance with the License. The rights granted to you under the License
 10   * may not be used to create, or enable the creation or redistribution of,
 11   * unlawful or unlicensed copies of an Apple operating system, or to
 12   * circumvent, violate, or enable the circumvention or violation of, any
 13   * terms of an Apple operating system software license agreement.
 14   *
 15   * Please obtain a copy of the License at
 16   * http://www.opensource.apple.com/apsl/ and read it before using this file.
 17   *
 18   * The Original Code and all software distributed under the License are
 19   * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 20   * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 21   * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 22   * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 23   * Please see the License for the specific language governing rights and
 24   * limitations under the License.
 25   *
 26   * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 27   */
 28  
 29  #ifndef _SYS_PTHREAD_INTERNAL_H_
 30  #define _SYS_PTHREAD_INTERNAL_H_
 31  
 32  #include <pthread/bsdthread_private.h>
 33  #include <pthread/priority_private.h>
 34  #include <pthread/workqueue_syscalls.h>
 35  
 36  #ifdef KERNEL
 37  struct ksyn_waitq_element;
 38  #include <stdatomic.h>
 39  #include <kern/thread_call.h>
 40  #include <kern/kcdata.h>
 41  #ifndef __DARLING__
 42  #include <sys/pthread_shims.h>
 43  #endif // __DARLING__
 44  #include <sys/queue.h>
 45  #include <sys/proc_info.h>
 46  
 47  #ifdef __arm64__
 48  #define PTHREAD_INLINE_RMW_ATOMICS 0
 49  #else
 50  #define PTHREAD_INLINE_RMW_ATOMICS 1
 51  #endif
 52  #endif // KERNEL
 53  
 54  #include "kern/synch_internal.h"
 55  #include "kern/workqueue_internal.h"
 56  #include "kern/kern_trace.h"
 57  #include "pthread/qos.h"
 58  #include "private/qos_private.h"
 59  
 60  /* pthread userspace SPI feature checking, these constants are returned from bsdthread_register,
 61   * as a bitmask, to inform userspace of the supported feature set. Old releases of OS X return
 62   * from this call either zero or -1, allowing us to return a positive number for feature bits.
 63   */
 64  #define PTHREAD_FEATURE_DISPATCHFUNC	0x01		/* same as WQOPS_QUEUE_NEWSPISUPP, checks for dispatch function support */
 65  #define PTHREAD_FEATURE_FINEPRIO		0x02		/* are fine grained prioirities available */
 66  #define PTHREAD_FEATURE_BSDTHREADCTL	0x04		/* is the bsdthread_ctl syscall available */
 67  #define PTHREAD_FEATURE_SETSELF			0x08		/* is the BSDTHREAD_CTL_SET_SELF command of bsdthread_ctl available */
 68  #define PTHREAD_FEATURE_QOS_MAINTENANCE	0x10		/* is QOS_CLASS_MAINTENANCE available */
 69  #define PTHREAD_FEATURE_RESERVED		0x20		/* burnt, shipped in OSX 10.11 & iOS 9 with partial kevent delivery support */
 70  #define PTHREAD_FEATURE_KEVENT          0x40		/* supports direct kevent delivery */
 71  #define PTHREAD_FEATURE_WORKLOOP          0x80		/* supports workloops */
 72  #define PTHREAD_FEATURE_QOS_DEFAULT		0x40000000	/* the kernel supports QOS_CLASS_DEFAULT */
 73  
 74  /* userspace <-> kernel registration struct, for passing data to/from the kext during main thread init. */
 75  struct _pthread_registration_data {
 76  	/*
 77  	 * version == sizeof(struct _pthread_registration_data)
 78  	 *
 79  	 * The structure can only grow, so we use its size as the version.
 80  	 * Userspace initializes this to the size of its structure and the kext
 81  	 * will copy out the version that was actually consumed.
 82  	 *
 83  	 * n.b. you must make sure the size of this structure isn't LP64-dependent
 84  	 */
 85  	uint64_t version;
 86  
 87  	uint64_t dispatch_queue_offset; /* copy-in */
 88  	uint64_t /* pthread_priority_t */ main_qos; /* copy-out */
 89  	uint32_t tsd_offset; /* copy-in */
 90  	uint32_t return_to_kernel_offset; /* copy-in */
 91  	uint32_t mach_thread_self_offset; /* copy-in */
 92  	mach_vm_address_t stack_addr_hint; /* copy-out */
 93  	uint32_t mutex_default_policy; /* copy-out */
 94  } __attribute__ ((packed));
 95  
 96  /*
 97   * "error" flags returned by fail condvar syscalls
 98   */
 99  #define ECVCLEARED	0x100
100  #define ECVPREPOST	0x200
101  
102  #ifdef KERNEL
103  
104  /* The set of features, from the feature bits above, that we support. */
105  #define PTHREAD_FEATURE_SUPPORTED	( \
106  	PTHREAD_FEATURE_DISPATCHFUNC | \
107  	PTHREAD_FEATURE_FINEPRIO | \
108  	PTHREAD_FEATURE_BSDTHREADCTL | \
109  	PTHREAD_FEATURE_SETSELF | \
110  	PTHREAD_FEATURE_QOS_MAINTENANCE | \
111  	PTHREAD_FEATURE_QOS_DEFAULT | \
112  	PTHREAD_FEATURE_KEVENT | \
113  	PTHREAD_FEATURE_WORKLOOP )
114  
115  #ifndef __DARLING__
116  extern pthread_callbacks_t pthread_kern;
117  #endif // __DARLING__
118  
119  struct ksyn_waitq_element {
120  	TAILQ_ENTRY(ksyn_waitq_element) kwe_list;	/* link to other list members */
121  	void *          kwe_kwqqueue;            	/* queue blocked on */
122  	thread_t        kwe_thread;
123  	uint16_t        kwe_state;			/* state */
124  	uint16_t        kwe_flags;
125  	uint32_t        kwe_lockseq;			/* the sequence of the entry */
126  	uint32_t	kwe_count;			/* upper bound on number of matches still pending */
127  	uint32_t 	kwe_psynchretval;		/* thread retval */
128  	void		*kwe_uth;			/* uthread */
129  };
130  typedef struct ksyn_waitq_element * ksyn_waitq_element_t;
131  
132  #ifdef __DARLING__
133  #include <sys/pthread_shims.h>
134  #endif // __DARLING__
135  
136  #ifdef __DARLING__
137  extern pthread_callbacks_t pthread_kern;
138  #endif // __DARLING__
139  
140  #define PTH_DEFAULT_STACKSIZE 512*1024
141  #define MAX_PTHREAD_SIZE 64*1024
142  
143  /* exported from the kernel but not present in any headers. */
144  extern thread_t port_name_to_thread(mach_port_name_t port_name);
145  
146  /* function declarations for pthread_kext.c */
147  void pthread_init(void);
148  void psynch_zoneinit(void);
149  void _pth_proc_hashinit(proc_t p);
150  void _pth_proc_hashdelete(proc_t p);
151  void pth_global_hashinit(void);
152  void psynch_wq_cleanup(void*, void*);
153  
154  void _pthread_init(void);
155  int _fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo);
156  uint32_t _get_pwq_state_kdp(proc_t p);
157  void _workqueue_exit(struct proc *p);
158  void _workqueue_mark_exiting(struct proc *p);
159  void _workqueue_thread_yielded(void);
160  sched_call_t _workqueue_get_sched_callback(void);
161  
162  int _bsdthread_create(struct proc *p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t *retval);
163  int _bsdthread_register(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t *retval);
164  int _bsdthread_terminate(struct proc *p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t *retval);
165  int _bsdthread_ctl_set_qos(struct proc *p, user_addr_t cmd, mach_port_name_t kport, user_addr_t tsd_priority_addr, user_addr_t arg3, int *retval);
166  int _bsdthread_ctl_set_self(struct proc *p, user_addr_t cmd, pthread_priority_t priority, mach_port_name_t voucher, _pthread_set_flags_t flags, int *retval);
167  int _bsdthread_ctl_qos_override_start(struct proc *p, user_addr_t cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int *retval);
168  int _bsdthread_ctl_qos_override_end(struct proc *p, user_addr_t cmd, mach_port_name_t kport, user_addr_t resource, user_addr_t arg3, int *retval);
169  int _bsdthread_ctl_qos_override_dispatch(struct proc __unused *p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t arg3, int __unused *retval);
170  int _bsdthread_ctl_qos_override_reset(struct proc __unused *p, user_addr_t __unused cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int __unused *retval);
171  int _bsdthread_ctl_qos_dispatch_asynchronous_override_add(struct proc __unused *p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int __unused *retval);
172  int _bsdthread_ctl_qos_dispatch_asynchronous_override_reset(struct proc __unused *p, user_addr_t __unused cmd, int reset_all, user_addr_t resource, user_addr_t arg3, int __unused *retval);
173  int _bsdthread_ctl(struct proc *p, user_addr_t cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int *retval);
174  int _thread_selfid(__unused struct proc *p, uint64_t *retval);
175  int _workq_kernreturn(struct proc *p, int options, user_addr_t item, int arg2, int arg3, int32_t *retval);
176  int _workq_open(struct proc *p, int32_t *retval);
177  
178  int _psynch_mutexwait(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
179  int _psynch_mutexdrop(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
180  int _psynch_cvbroad(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex,  uint64_t mugen, uint64_t tid, uint32_t *retval);
181  int _psynch_cvsignal(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex,  uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t * retval);
182  int _psynch_cvwait(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval);
183  int _psynch_cvclrprepost(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval);
184  int _psynch_rw_longrdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t * retval);
185  int _psynch_rw_rdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
186  int _psynch_rw_unlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
187  int _psynch_rw_wrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
188  int _psynch_rw_yieldwrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
189  
190  void _pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
191  void * _pthread_get_thread_kwq(thread_t thread);
192  
193  extern lck_grp_attr_t *pthread_lck_grp_attr;
194  extern lck_grp_t *pthread_lck_grp;
195  extern lck_attr_t *pthread_lck_attr;
196  extern lck_mtx_t *pthread_list_mlock;
197  extern thread_call_t psynch_thcall;
198  
199  struct uthread* current_uthread(void);
200  
201  int
202  workq_create_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t *out_addr);
203  
204  int
205  workq_destroy_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t stackaddr);
206  
207  void
208  workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr,
209  		mach_port_name_t kport, int th_qos, int setup_flags, int upcall_flags);
210  
211  int
212  workq_handle_stack_events(proc_t p, thread_t th, vm_map_t map,
213  		user_addr_t stackaddr, mach_port_name_t kport,
214  		user_addr_t events, int nevents, int upcall_flags);
215  
216  void
217  workq_markfree_threadstack(proc_t p, thread_t th, vm_map_t vmap,
218  		user_addr_t stackaddr);
219  
220  #endif // KERNEL
221  
222  #endif /* _SYS_PTHREAD_INTERNAL_H_ */
223