thread.c
1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 #include <assert.h> 4 #include <bootstate.h> 5 #include <console/console.h> 6 #include <smp/node.h> 7 #include <thread.h> 8 #include <timer.h> 9 #include <types.h> 10 11 static u8 thread_stacks[CONFIG_STACK_SIZE * CONFIG_NUM_THREADS] __aligned(sizeof(uint64_t)); 12 static bool initialized; 13 14 static void idle_thread_init(void); 15 16 /* There needs to be at least one thread to run the ramstate state machine. */ 17 #define TOTAL_NUM_THREADS (CONFIG_NUM_THREADS + 1) 18 19 /* Storage space for the thread structs .*/ 20 static struct thread all_threads[TOTAL_NUM_THREADS]; 21 22 /* All runnable (but not running) and free threads are kept on their 23 * respective lists. */ 24 static struct thread *runnable_threads; 25 static struct thread *free_threads; 26 27 static struct thread *active_thread; 28 29 static inline int thread_can_yield(const struct thread *t) 30 { 31 return (t != NULL && t->can_yield > 0); 32 } 33 34 static inline void set_current_thread(struct thread *t) 35 { 36 assert(boot_cpu()); 37 active_thread = t; 38 } 39 40 static inline struct thread *current_thread(void) 41 { 42 if (!initialized || !boot_cpu()) 43 return NULL; 44 45 return active_thread; 46 } 47 48 static inline int thread_list_empty(struct thread **list) 49 { 50 return *list == NULL; 51 } 52 53 static inline struct thread *pop_thread(struct thread **list) 54 { 55 struct thread *t; 56 57 t = *list; 58 *list = t->next; 59 t->next = NULL; 60 return t; 61 } 62 63 static inline void push_thread(struct thread **list, struct thread *t) 64 { 65 t->next = *list; 66 *list = t; 67 } 68 69 static inline void push_runnable(struct thread *t) 70 { 71 push_thread(&runnable_threads, t); 72 } 73 74 static inline struct thread *pop_runnable(void) 75 { 76 return pop_thread(&runnable_threads); 77 } 78 79 static inline struct thread *get_free_thread(void) 80 { 81 struct thread *t; 82 83 if (thread_list_empty(&free_threads)) 84 return NULL; 85 86 t = pop_thread(&free_threads); 87 88 /* Reset the current stack value to the original. */ 89 if (!t->stack_orig) 90 die("%s: Invalid stack value\n", __func__); 91 92 t->stack_current = t->stack_orig; 93 94 return t; 95 } 96 97 static inline void free_thread(struct thread *t) 98 { 99 push_thread(&free_threads, t); 100 } 101 102 /* The idle thread is ran whenever there isn't anything else that is runnable. 103 * It's sole responsibility is to ensure progress is made by running the timer 104 * callbacks. */ 105 __noreturn static enum cb_err idle_thread(void *unused) 106 { 107 /* This thread never voluntarily yields. */ 108 thread_coop_disable(); 109 while (1) 110 timers_run(); 111 } 112 113 static void schedule(struct thread *t) 114 { 115 struct thread *current = current_thread(); 116 117 /* If t is NULL need to find new runnable thread. */ 118 if (t == NULL) { 119 if (thread_list_empty(&runnable_threads)) 120 die("Runnable thread list is empty!\n"); 121 t = pop_runnable(); 122 } else { 123 /* current is still runnable. */ 124 push_runnable(current); 125 } 126 127 if (t->handle) 128 t->handle->state = THREAD_STARTED; 129 130 set_current_thread(t); 131 132 switch_to_thread(t->stack_current, ¤t->stack_current); 133 } 134 135 static void terminate_thread(struct thread *t, enum cb_err error) 136 { 137 if (t->handle) { 138 t->handle->error = error; 139 t->handle->state = THREAD_DONE; 140 } 141 142 free_thread(t); 143 schedule(NULL); 144 } 145 146 static asmlinkage void call_wrapper(void *unused) 147 { 148 struct thread *current = current_thread(); 149 enum cb_err error; 150 151 error = current->entry(current->entry_arg); 152 153 terminate_thread(current, error); 154 } 155 156 struct block_boot_state { 157 boot_state_t state; 158 boot_state_sequence_t seq; 159 }; 160 161 /* Block the provided state until thread is complete. */ 162 static asmlinkage void call_wrapper_block_state(void *arg) 163 { 164 struct block_boot_state *bbs = arg; 165 struct thread *current = current_thread(); 166 enum cb_err error; 167 168 boot_state_block(bbs->state, bbs->seq); 169 error = current->entry(current->entry_arg); 170 boot_state_unblock(bbs->state, bbs->seq); 171 terminate_thread(current, error); 172 } 173 174 /* Prepare a thread so that it starts by executing thread_entry(thread_arg). 175 * Within thread_entry() it will call func(arg). */ 176 static void prepare_thread(struct thread *t, struct thread_handle *handle, 177 enum cb_err (*func)(void *), void *arg, 178 asmlinkage void (*thread_entry)(void *), void *thread_arg) 179 { 180 /* Stash the function and argument to run. */ 181 t->entry = func; 182 t->entry_arg = arg; 183 184 /* All new threads can yield by default. */ 185 t->can_yield = 1; 186 187 /* Pointer used to publish the state of thread */ 188 t->handle = handle; 189 190 arch_prepare_thread(t, thread_entry, thread_arg); 191 } 192 193 static void thread_resume_from_timeout(struct timeout_callback *tocb) 194 { 195 struct thread *to; 196 197 to = tocb->priv; 198 schedule(to); 199 } 200 201 static void idle_thread_init(void) 202 { 203 struct thread *t; 204 205 t = get_free_thread(); 206 207 if (t == NULL) 208 die("No threads available for idle thread!\n"); 209 210 /* Queue idle thread to run once all other threads have yielded. */ 211 prepare_thread(t, NULL, idle_thread, NULL, call_wrapper, NULL); 212 push_runnable(t); 213 } 214 215 /* Don't inline this function so the timeout_callback won't have its storage 216 * space on the stack cleaned up before the call to schedule(). */ 217 static int __attribute__((noinline)) 218 thread_yield_timed_callback(struct timeout_callback *tocb, 219 unsigned int microsecs) 220 { 221 tocb->priv = current_thread(); 222 tocb->callback = thread_resume_from_timeout; 223 224 if (timer_sched_callback(tocb, microsecs)) 225 return -1; 226 227 /* The timer callback will wake up the current thread. */ 228 schedule(NULL); 229 return 0; 230 } 231 232 static void *thread_alloc_space(struct thread *t, size_t bytes) 233 { 234 /* Allocate the amount of space on the stack keeping the stack 235 * aligned to the pointer size. */ 236 t->stack_current -= ALIGN_UP(bytes, sizeof(uintptr_t)); 237 238 return (void *)t->stack_current; 239 } 240 241 static void threads_initialize(void) 242 { 243 int i; 244 struct thread *t; 245 u8 *stack_top; 246 247 if (initialized) 248 return; 249 250 t = &all_threads[0]; 251 252 set_current_thread(t); 253 254 t->stack_orig = 0; /* We never free the main thread */ 255 t->id = 0; 256 t->can_yield = 1; 257 258 stack_top = &thread_stacks[CONFIG_STACK_SIZE]; 259 for (i = 1; i < TOTAL_NUM_THREADS; i++) { 260 t = &all_threads[i]; 261 t->stack_orig = (uintptr_t)stack_top; 262 t->id = i; 263 stack_top += CONFIG_STACK_SIZE; 264 free_thread(t); 265 } 266 267 idle_thread_init(); 268 269 initialized = 1; 270 } 271 272 int thread_run(struct thread_handle *handle, enum cb_err (*func)(void *), void *arg) 273 { 274 struct thread *current; 275 struct thread *t; 276 277 /* Lazy initialization */ 278 threads_initialize(); 279 280 current = current_thread(); 281 282 if (!thread_can_yield(current)) { 283 printk(BIOS_ERR, "%s() called from non-yielding context!\n", __func__); 284 return -1; 285 } 286 287 t = get_free_thread(); 288 289 if (t == NULL) { 290 printk(BIOS_ERR, "%s: No more threads!\n", __func__); 291 return -1; 292 } 293 294 prepare_thread(t, handle, func, arg, call_wrapper, NULL); 295 schedule(t); 296 297 return 0; 298 } 299 300 int thread_run_until(struct thread_handle *handle, enum cb_err (*func)(void *), void *arg, 301 boot_state_t state, boot_state_sequence_t seq) 302 { 303 struct thread *current; 304 struct thread *t; 305 struct block_boot_state *bbs; 306 307 /* This is a ramstage specific API */ 308 if (!ENV_RAMSTAGE) 309 dead_code(); 310 311 /* Lazy initialization */ 312 threads_initialize(); 313 314 current = current_thread(); 315 316 if (!thread_can_yield(current)) { 317 printk(BIOS_ERR, "%s() called from non-yielding context!\n", __func__); 318 return -1; 319 } 320 321 t = get_free_thread(); 322 323 if (t == NULL) { 324 printk(BIOS_ERR, "%s: No more threads!\n", __func__); 325 return -1; 326 } 327 328 bbs = thread_alloc_space(t, sizeof(*bbs)); 329 bbs->state = state; 330 bbs->seq = seq; 331 prepare_thread(t, handle, func, arg, call_wrapper_block_state, bbs); 332 schedule(t); 333 334 return 0; 335 } 336 337 int thread_yield(void) 338 { 339 return thread_yield_microseconds(0); 340 } 341 342 int thread_yield_microseconds(unsigned int microsecs) 343 { 344 struct thread *current; 345 struct timeout_callback tocb; 346 347 current = current_thread(); 348 349 if (!thread_can_yield(current)) 350 return -1; 351 352 if (thread_yield_timed_callback(&tocb, microsecs)) 353 return -1; 354 355 return 0; 356 } 357 358 void thread_coop_enable(void) 359 { 360 struct thread *current; 361 362 current = current_thread(); 363 364 if (current == NULL) 365 return; 366 367 assert(current->can_yield <= 0); 368 369 current->can_yield++; 370 } 371 372 void thread_coop_disable(void) 373 { 374 struct thread *current; 375 376 current = current_thread(); 377 378 if (current == NULL) 379 return; 380 381 current->can_yield--; 382 } 383 384 enum cb_err thread_join(struct thread_handle *handle) 385 { 386 struct stopwatch sw; 387 struct thread *current = current_thread(); 388 389 assert(handle); 390 assert(current); 391 assert(current->handle != handle); 392 393 if (handle->state == THREAD_UNINITIALIZED) 394 return CB_ERR_ARG; 395 396 printk(BIOS_SPEW, "waiting for thread\n"); 397 398 stopwatch_init(&sw); 399 400 while (handle->state != THREAD_DONE) 401 assert(thread_yield() == 0); 402 403 printk(BIOS_SPEW, "took %lld us\n", stopwatch_duration_usecs(&sw)); 404 405 return handle->error; 406 } 407 408 void thread_mutex_lock(struct thread_mutex *mutex) 409 { 410 struct stopwatch sw; 411 412 stopwatch_init(&sw); 413 414 while (mutex->locked) 415 assert(thread_yield() == 0); 416 mutex->locked = true; 417 418 printk(BIOS_SPEW, "took %lld us to acquire mutex\n", stopwatch_duration_usecs(&sw)); 419 } 420 421 void thread_mutex_unlock(struct thread_mutex *mutex) 422 { 423 assert(mutex->locked); 424 mutex->locked = 0; 425 }