init.c
1 /* ---------------------------------------------------------------------------- 2 Copyright (c) 2018-2022, Microsoft Research, Daan Leijen 3 This is free software; you can redistribute it and/or modify it under the 4 terms of the MIT license. A copy of the license can be found in the file 5 "LICENSE" at the root of this distribution. 6 -----------------------------------------------------------------------------*/ 7 #include "mimalloc.h" 8 #include "mimalloc/internal.h" 9 #include "mimalloc/prim.h" 10 11 #include <string.h> // memcpy, memset 12 #include <stdlib.h> // atexit 13 14 15 // Empty page used to initialize the small free pages array 16 const mi_page_t _mi_page_empty = { 17 0, 18 false, false, false, false, 19 0, // capacity 20 0, // reserved capacity 21 { 0 }, // flags 22 false, // is_zero 23 0, // retire_expire 24 NULL, // free 25 NULL, // local_free 26 0, // used 27 0, // block size shift 28 0, // heap tag 29 0, // block_size 30 NULL, // page_start 31 #if (MI_PADDING || MI_ENCODE_FREELIST) 32 { 0, 0 }, 33 #endif 34 MI_ATOMIC_VAR_INIT(0), // xthread_free 35 MI_ATOMIC_VAR_INIT(0), // xheap 36 NULL, NULL 37 #if MI_INTPTR_SIZE==4 38 , { NULL } 39 #endif 40 }; 41 42 #define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) 43 44 #if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) 45 #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } 46 #elif (MI_PADDING>0) 47 #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } 48 #else 49 #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } 50 #endif 51 52 53 // Empty page queues for every bin 54 #define QNULL(sz) { NULL, NULL, (sz)*sizeof(uintptr_t) } 55 #define MI_PAGE_QUEUES_EMPTY \ 56 { QNULL(1), \ 57 QNULL( 1), QNULL( 2), QNULL( 3), QNULL( 4), QNULL( 5), QNULL( 6), QNULL( 7), QNULL( 8), /* 8 */ \ 58 QNULL( 10), QNULL( 12), QNULL( 14), QNULL( 16), QNULL( 20), QNULL( 24), QNULL( 28), QNULL( 32), /* 16 */ \ 59 QNULL( 40), QNULL( 48), QNULL( 56), QNULL( 64), QNULL( 80), QNULL( 96), QNULL( 112), QNULL( 128), /* 24 */ \ 60 QNULL( 160), QNULL( 192), QNULL( 224), QNULL( 256), QNULL( 320), QNULL( 384), QNULL( 448), QNULL( 512), /* 32 */ \ 61 QNULL( 640), QNULL( 768), QNULL( 896), QNULL( 1024), QNULL( 1280), QNULL( 1536), QNULL( 1792), QNULL( 2048), /* 40 */ \ 62 QNULL( 2560), QNULL( 3072), QNULL( 3584), QNULL( 4096), QNULL( 5120), QNULL( 6144), QNULL( 7168), QNULL( 8192), /* 48 */ \ 63 QNULL( 10240), QNULL( 12288), QNULL( 14336), QNULL( 16384), QNULL( 20480), QNULL( 24576), QNULL( 28672), QNULL( 32768), /* 56 */ \ 64 QNULL( 40960), QNULL( 49152), QNULL( 57344), QNULL( 65536), QNULL( 81920), QNULL( 98304), QNULL(114688), QNULL(131072), /* 64 */ \ 65 QNULL(163840), QNULL(196608), QNULL(229376), QNULL(262144), QNULL(327680), QNULL(393216), QNULL(458752), QNULL(524288), /* 72 */ \ 66 QNULL(MI_LARGE_OBJ_WSIZE_MAX + 1 /* 655360, Huge queue */), \ 67 QNULL(MI_LARGE_OBJ_WSIZE_MAX + 2) /* Full queue */ } 68 69 #define MI_STAT_COUNT_NULL() {0,0,0,0} 70 71 // Empty statistics 72 #if MI_STAT>1 73 #define MI_STAT_COUNT_END_NULL() , { MI_STAT_COUNT_NULL(), MI_INIT32(MI_STAT_COUNT_NULL) } 74 #else 75 #define MI_STAT_COUNT_END_NULL() 76 #endif 77 78 #define MI_STATS_NULL \ 79 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 80 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 81 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 82 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 83 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 84 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 85 MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \ 86 MI_STAT_COUNT_NULL(), \ 87 { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ 88 { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, \ 89 { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } \ 90 MI_STAT_COUNT_END_NULL() 91 92 // -------------------------------------------------------- 93 // Statically allocate an empty heap as the initial 94 // thread local value for the default heap, 95 // and statically allocate the backing heap for the main 96 // thread so it can function without doing any allocation 97 // itself (as accessing a thread local for the first time 98 // may lead to allocation itself on some platforms) 99 // -------------------------------------------------------- 100 101 mi_decl_cache_align const mi_heap_t _mi_heap_empty = { 102 NULL, 103 MI_ATOMIC_VAR_INIT(NULL), 104 0, // tid 105 0, // cookie 106 0, // arena id 107 { 0, 0 }, // keys 108 { {0}, {0}, 0, true }, // random 109 0, // page count 110 MI_BIN_FULL, 0, // page retired min/max 111 NULL, // next 112 false, // can reclaim 113 0, // tag 114 MI_SMALL_PAGES_EMPTY, 115 MI_PAGE_QUEUES_EMPTY 116 }; 117 118 119 mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { 120 return _mi_prim_thread_id(); 121 } 122 123 // the thread-local default heap for allocation 124 mi_decl_thread mi_heap_t* _mi_heap_default = (mi_heap_t*)&_mi_heap_empty; 125 126 extern mi_heap_t _mi_heap_main; 127 128 static mi_decl_cache_align mi_subproc_t mi_subproc_default; 129 130 static mi_decl_cache_align mi_tld_t tld_main = { 131 0, false, 132 &_mi_heap_main, &_mi_heap_main, 133 { { NULL, NULL }, {NULL ,NULL}, {NULL ,NULL, 0}, 134 0, 0, 0, 0, 0, &mi_subproc_default, 135 &tld_main.stats, &tld_main.os 136 }, // segments 137 { 0, &tld_main.stats }, // os 138 { MI_STATS_NULL } // stats 139 }; 140 141 mi_decl_cache_align mi_heap_t _mi_heap_main = { 142 &tld_main, 143 MI_ATOMIC_VAR_INIT(NULL), 144 0, // thread id 145 0, // initial cookie 146 0, // arena id 147 { 0, 0 }, // the key of the main heap can be fixed (unlike page keys that need to be secure!) 148 { {0x846ca68b}, {0}, 0, true }, // random 149 0, // page count 150 MI_BIN_FULL, 0, // page retired min/max 151 NULL, // next heap 152 false, // can reclaim 153 0, // tag 154 MI_SMALL_PAGES_EMPTY, 155 MI_PAGE_QUEUES_EMPTY 156 }; 157 158 bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. 159 160 mi_stats_t _mi_stats_main = { MI_STATS_NULL }; 161 162 163 static void mi_heap_main_init(void) { 164 if (_mi_heap_main.cookie == 0) { 165 _mi_heap_main.thread_id = _mi_thread_id(); 166 _mi_heap_main.cookie = 1; 167 #if defined(_WIN32) && !defined(MI_SHARED_LIB) 168 _mi_random_init_weak(&_mi_heap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking 169 #else 170 _mi_random_init(&_mi_heap_main.random); 171 #endif 172 _mi_heap_main.cookie = _mi_heap_random_next(&_mi_heap_main); 173 _mi_heap_main.keys[0] = _mi_heap_random_next(&_mi_heap_main); 174 _mi_heap_main.keys[1] = _mi_heap_random_next(&_mi_heap_main); 175 mi_lock_init(&mi_subproc_default.abandoned_os_lock); 176 mi_lock_init(&mi_subproc_default.abandoned_os_visit_lock); 177 } 178 } 179 180 mi_heap_t* _mi_heap_main_get(void) { 181 mi_heap_main_init(); 182 return &_mi_heap_main; 183 } 184 185 186 /* ----------------------------------------------------------- 187 Sub process 188 ----------------------------------------------------------- */ 189 190 mi_subproc_id_t mi_subproc_main(void) { 191 return NULL; 192 } 193 194 mi_subproc_id_t mi_subproc_new(void) { 195 mi_memid_t memid = _mi_memid_none(); 196 mi_subproc_t* subproc = (mi_subproc_t*)_mi_arena_meta_zalloc(sizeof(mi_subproc_t), &memid); 197 if (subproc == NULL) return NULL; 198 subproc->memid = memid; 199 subproc->abandoned_os_list = NULL; 200 mi_lock_init(&subproc->abandoned_os_lock); 201 mi_lock_init(&subproc->abandoned_os_visit_lock); 202 return subproc; 203 } 204 205 mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { 206 return (subproc_id == NULL ? &mi_subproc_default : (mi_subproc_t*)subproc_id); 207 } 208 209 void mi_subproc_delete(mi_subproc_id_t subproc_id) { 210 if (subproc_id == NULL) return; 211 mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); 212 // check if there are no abandoned segments still.. 213 bool safe_to_delete = false; 214 if (mi_lock_acquire(&subproc->abandoned_os_lock)) { 215 if (subproc->abandoned_os_list == NULL) { 216 safe_to_delete = true; 217 } 218 mi_lock_release(&subproc->abandoned_os_lock); 219 } 220 if (!safe_to_delete) return; 221 // safe to release 222 // todo: should we refcount subprocesses? 223 mi_lock_done(&subproc->abandoned_os_lock); 224 mi_lock_done(&subproc->abandoned_os_visit_lock); 225 _mi_arena_meta_free(subproc, subproc->memid, sizeof(mi_subproc_t)); 226 } 227 228 void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { 229 mi_heap_t* heap = mi_heap_get_default(); 230 if (heap == NULL) return; 231 mi_assert(heap->tld->segments.subproc == &mi_subproc_default); 232 if (heap->tld->segments.subproc != &mi_subproc_default) return; 233 heap->tld->segments.subproc = _mi_subproc_from_id(subproc_id); 234 } 235 236 237 238 /* ----------------------------------------------------------- 239 Initialization and freeing of the thread local heaps 240 ----------------------------------------------------------- */ 241 242 // note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). 243 typedef struct mi_thread_data_s { 244 mi_heap_t heap; // must come first due to cast in `_mi_heap_done` 245 mi_tld_t tld; 246 mi_memid_t memid; // must come last due to zero'ing 247 } mi_thread_data_t; 248 249 250 // Thread meta-data is allocated directly from the OS. For 251 // some programs that do not use thread pools and allocate and 252 // destroy many OS threads, this may causes too much overhead 253 // per thread so we maintain a small cache of recently freed metadata. 254 255 #define TD_CACHE_SIZE (32) 256 static _Atomic(mi_thread_data_t*) td_cache[TD_CACHE_SIZE]; 257 258 static mi_thread_data_t* mi_thread_data_zalloc(void) { 259 // try to find thread metadata in the cache 260 bool is_zero = false; 261 mi_thread_data_t* td = NULL; 262 for (int i = 0; i < TD_CACHE_SIZE; i++) { 263 td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); 264 if (td != NULL) { 265 // found cached allocation, try use it 266 td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); 267 if (td != NULL) { 268 break; 269 } 270 } 271 } 272 273 // if that fails, allocate as meta data 274 if (td == NULL) { 275 mi_memid_t memid; 276 td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); 277 if (td == NULL) { 278 // if this fails, try once more. (issue #257) 279 td = (mi_thread_data_t*)_mi_os_alloc(sizeof(mi_thread_data_t), &memid, &_mi_stats_main); 280 if (td == NULL) { 281 // really out of memory 282 _mi_error_message(ENOMEM, "unable to allocate thread local heap metadata (%zu bytes)\n", sizeof(mi_thread_data_t)); 283 } 284 } 285 if (td != NULL) { 286 td->memid = memid; 287 is_zero = memid.initially_zero; 288 } 289 } 290 291 if (td != NULL && !is_zero) { 292 _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid)); 293 } 294 return td; 295 } 296 297 static void mi_thread_data_free( mi_thread_data_t* tdfree ) { 298 // try to add the thread metadata to the cache 299 for (int i = 0; i < TD_CACHE_SIZE; i++) { 300 mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); 301 if (td == NULL) { 302 mi_thread_data_t* expected = NULL; 303 if (mi_atomic_cas_ptr_weak_acq_rel(mi_thread_data_t, &td_cache[i], &expected, tdfree)) { 304 return; 305 } 306 } 307 } 308 // if that fails, just free it directly 309 _mi_os_free(tdfree, sizeof(mi_thread_data_t), tdfree->memid, &_mi_stats_main); 310 } 311 312 void _mi_thread_data_collect(void) { 313 // free all thread metadata from the cache 314 for (int i = 0; i < TD_CACHE_SIZE; i++) { 315 mi_thread_data_t* td = mi_atomic_load_ptr_relaxed(mi_thread_data_t, &td_cache[i]); 316 if (td != NULL) { 317 td = mi_atomic_exchange_ptr_acq_rel(mi_thread_data_t, &td_cache[i], NULL); 318 if (td != NULL) { 319 _mi_os_free(td, sizeof(mi_thread_data_t), td->memid, &_mi_stats_main); 320 } 321 } 322 } 323 } 324 325 // Initialize the thread local default heap, called from `mi_thread_init` 326 static bool _mi_thread_heap_init(void) { 327 if (mi_heap_is_initialized(mi_prim_get_default_heap())) return true; 328 if (_mi_is_main_thread()) { 329 // mi_assert_internal(_mi_heap_main.thread_id != 0); // can happen on freeBSD where alloc is called before any initialization 330 // the main heap is statically allocated 331 mi_heap_main_init(); 332 _mi_heap_set_default_direct(&_mi_heap_main); 333 //mi_assert_internal(_mi_heap_default->tld->heap_backing == mi_prim_get_default_heap()); 334 } 335 else { 336 // use `_mi_os_alloc` to allocate directly from the OS 337 mi_thread_data_t* td = mi_thread_data_zalloc(); 338 if (td == NULL) return false; 339 340 mi_tld_t* tld = &td->tld; 341 mi_heap_t* heap = &td->heap; 342 _mi_tld_init(tld, heap); // must be before `_mi_heap_init` 343 _mi_heap_init(heap, tld, _mi_arena_id_none(), false /* can reclaim */, 0 /* default tag */); 344 _mi_heap_set_default_direct(heap); 345 } 346 return false; 347 } 348 349 // initialize thread local data 350 void _mi_tld_init(mi_tld_t* tld, mi_heap_t* bheap) { 351 _mi_memzero_aligned(tld,sizeof(mi_tld_t)); 352 tld->heap_backing = bheap; 353 tld->heaps = NULL; 354 tld->segments.subproc = &mi_subproc_default; 355 tld->segments.stats = &tld->stats; 356 tld->segments.os = &tld->os; 357 tld->os.stats = &tld->stats; 358 } 359 360 // Free the thread local default heap (called from `mi_thread_done`) 361 static bool _mi_thread_heap_done(mi_heap_t* heap) { 362 if (!mi_heap_is_initialized(heap)) return true; 363 364 // reset default heap 365 _mi_heap_set_default_direct(_mi_is_main_thread() ? &_mi_heap_main : (mi_heap_t*)&_mi_heap_empty); 366 367 // switch to backing heap 368 heap = heap->tld->heap_backing; 369 if (!mi_heap_is_initialized(heap)) return false; 370 371 // delete all non-backing heaps in this thread 372 mi_heap_t* curr = heap->tld->heaps; 373 while (curr != NULL) { 374 mi_heap_t* next = curr->next; // save `next` as `curr` will be freed 375 if (curr != heap) { 376 mi_assert_internal(!mi_heap_is_backing(curr)); 377 mi_heap_delete(curr); 378 } 379 curr = next; 380 } 381 mi_assert_internal(heap->tld->heaps == heap && heap->next == NULL); 382 mi_assert_internal(mi_heap_is_backing(heap)); 383 384 // collect if not the main thread 385 if (heap != &_mi_heap_main) { 386 _mi_heap_collect_abandon(heap); 387 } 388 389 // merge stats 390 _mi_stats_done(&heap->tld->stats); 391 392 // free if not the main thread 393 if (heap != &_mi_heap_main) { 394 mi_assert_internal(heap->tld->segments.count == 0 || heap->thread_id != _mi_thread_id()); 395 mi_thread_data_free((mi_thread_data_t*)heap); 396 } 397 else { 398 #if 0 399 // never free the main thread even in debug mode; if a dll is linked statically with mimalloc, 400 // there may still be delete/free calls after the mi_fls_done is called. Issue #207 401 _mi_heap_destroy_pages(heap); 402 mi_assert_internal(heap->tld->heap_backing == &_mi_heap_main); 403 #endif 404 } 405 return false; 406 } 407 408 409 410 // -------------------------------------------------------- 411 // Try to run `mi_thread_done()` automatically so any memory 412 // owned by the thread but not yet released can be abandoned 413 // and re-owned by another thread. 414 // 415 // 1. windows dynamic library: 416 // call from DllMain on DLL_THREAD_DETACH 417 // 2. windows static library: 418 // use `FlsAlloc` to call a destructor when the thread is done 419 // 3. unix, pthreads: 420 // use a pthread key to call a destructor when a pthread is done 421 // 422 // In the last two cases we also need to call `mi_process_init` 423 // to set up the thread local keys. 424 // -------------------------------------------------------- 425 426 // Set up handlers so `mi_thread_done` is called automatically 427 static void mi_process_setup_auto_thread_done(void) { 428 static bool tls_initialized = false; // fine if it races 429 if (tls_initialized) return; 430 tls_initialized = true; 431 _mi_prim_thread_init_auto_done(); 432 _mi_heap_set_default_direct(&_mi_heap_main); 433 } 434 435 436 bool _mi_is_main_thread(void) { 437 return (_mi_heap_main.thread_id==0 || _mi_heap_main.thread_id == _mi_thread_id()); 438 } 439 440 static _Atomic(size_t) thread_count = MI_ATOMIC_VAR_INIT(1); 441 442 size_t _mi_current_thread_count(void) { 443 return mi_atomic_load_relaxed(&thread_count); 444 } 445 446 // This is called from the `mi_malloc_generic` 447 void mi_thread_init(void) mi_attr_noexcept 448 { 449 // ensure our process has started already 450 mi_process_init(); 451 452 // initialize the thread local default heap 453 // (this will call `_mi_heap_set_default_direct` and thus set the 454 // fiber/pthread key to a non-zero value, ensuring `_mi_thread_done` is called) 455 if (_mi_thread_heap_init()) return; // returns true if already initialized 456 457 _mi_stat_increase(&_mi_stats_main.threads, 1); 458 mi_atomic_increment_relaxed(&thread_count); 459 //_mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); 460 } 461 462 void mi_thread_done(void) mi_attr_noexcept { 463 _mi_thread_done(NULL); 464 } 465 466 void _mi_thread_done(mi_heap_t* heap) 467 { 468 // calling with NULL implies using the default heap 469 if (heap == NULL) { 470 heap = mi_prim_get_default_heap(); 471 if (heap == NULL) return; 472 } 473 474 // prevent re-entrancy through heap_done/heap_set_default_direct (issue #699) 475 if (!mi_heap_is_initialized(heap)) { 476 return; 477 } 478 479 // adjust stats 480 mi_atomic_decrement_relaxed(&thread_count); 481 _mi_stat_decrease(&_mi_stats_main.threads, 1); 482 483 // check thread-id as on Windows shutdown with FLS the main (exit) thread may call this on thread-local heaps... 484 if (heap->thread_id != _mi_thread_id()) return; 485 486 // abandon the thread local heap 487 if (_mi_thread_heap_done(heap)) return; // returns true if already ran 488 } 489 490 void _mi_heap_set_default_direct(mi_heap_t* heap) { 491 mi_assert_internal(heap != NULL); 492 #if defined(MI_TLS_SLOT) 493 mi_prim_tls_slot_set(MI_TLS_SLOT,heap); 494 #elif defined(MI_TLS_PTHREAD_SLOT_OFS) 495 *mi_prim_tls_pthread_heap_slot() = heap; 496 #elif defined(MI_TLS_PTHREAD) 497 // we use _mi_heap_default_key 498 #else 499 _mi_heap_default = heap; 500 #endif 501 502 // ensure the default heap is passed to `_mi_thread_done` 503 // setting to a non-NULL value also ensures `mi_thread_done` is called. 504 _mi_prim_thread_associate_default_heap(heap); 505 } 506 507 508 // -------------------------------------------------------- 509 // Run functions on process init/done, and thread init/done 510 // -------------------------------------------------------- 511 static bool os_preloading = true; // true until this module is initialized 512 513 // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. 514 bool mi_decl_noinline _mi_preloading(void) { 515 return os_preloading; 516 } 517 518 // Called once by the process loader from `src/prim/prim.c` 519 void _mi_process_load(void) { 520 mi_heap_main_init(); 521 #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) 522 volatile mi_heap_t* dummy = _mi_heap_default; // access TLS to allocate it before setting tls_initialized to true; 523 if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697) 524 #endif 525 os_preloading = false; 526 mi_assert_internal(_mi_is_main_thread()); 527 _mi_options_init(); 528 mi_process_setup_auto_thread_done(); 529 mi_process_init(); 530 if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); 531 532 // show message from the redirector (if present) 533 const char* msg = NULL; 534 _mi_allocator_init(&msg); 535 if (msg != NULL && (mi_option_is_enabled(mi_option_verbose) || mi_option_is_enabled(mi_option_show_errors))) { 536 _mi_fputs(NULL,NULL,NULL,msg); 537 } 538 539 // reseed random 540 _mi_random_reinit_if_weak(&_mi_heap_main.random); 541 } 542 543 #if defined(_WIN32) && (defined(_M_IX86) || defined(_M_X64)) 544 #include <intrin.h> 545 mi_decl_cache_align bool _mi_cpu_has_fsrm = false; 546 547 static void mi_detect_cpu_features(void) { 548 // FSRM for fast rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) 549 int32_t cpu_info[4]; 550 __cpuid(cpu_info, 7); 551 _mi_cpu_has_fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see <https://en.wikipedia.org/wiki/CPUID#EAX=7,_ECX=0:_Extended_Features> 552 } 553 #else 554 static void mi_detect_cpu_features(void) { 555 // nothing 556 } 557 #endif 558 559 // Initialize the process; called by thread_init or the process loader 560 void mi_process_init(void) mi_attr_noexcept { 561 // ensure we are called once 562 static mi_atomic_once_t process_init; 563 #if _MSC_VER < 1920 564 mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main 565 #endif 566 if (!mi_atomic_once(&process_init)) return; 567 _mi_process_is_initialized = true; 568 _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); 569 mi_process_setup_auto_thread_done(); 570 571 mi_detect_cpu_features(); 572 _mi_os_init(); 573 mi_heap_main_init(); 574 #if MI_DEBUG 575 _mi_verbose_message("debug level : %d\n", MI_DEBUG); 576 #endif 577 _mi_verbose_message("secure level: %d\n", MI_SECURE); 578 _mi_verbose_message("mem tracking: %s\n", MI_TRACK_TOOL); 579 #if MI_TSAN 580 _mi_verbose_message("thread santizer enabled\n"); 581 #endif 582 mi_thread_init(); 583 584 #if defined(_WIN32) 585 // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. 586 // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup 587 // will not call _mi_thread_done on the (still executing) main thread. See issue #508. 588 _mi_prim_thread_associate_default_heap(NULL); 589 #endif 590 591 mi_stats_reset(); // only call stat reset *after* thread init (or the heap tld == NULL) 592 mi_track_init(); 593 594 if (mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { 595 size_t pages = mi_option_get_clamp(mi_option_reserve_huge_os_pages, 0, 128*1024); 596 long reserve_at = mi_option_get(mi_option_reserve_huge_os_pages_at); 597 if (reserve_at != -1) { 598 mi_reserve_huge_os_pages_at(pages, reserve_at, pages*500); 599 } else { 600 mi_reserve_huge_os_pages_interleave(pages, 0, pages*500); 601 } 602 } 603 if (mi_option_is_enabled(mi_option_reserve_os_memory)) { 604 long ksize = mi_option_get(mi_option_reserve_os_memory); 605 if (ksize > 0) { 606 mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); 607 } 608 } 609 } 610 611 // Called when the process is done (through `at_exit`) 612 void mi_cdecl _mi_process_done(void) { 613 // only shutdown if we were initialized 614 if (!_mi_process_is_initialized) return; 615 // ensure we are called once 616 static bool process_done = false; 617 if (process_done) return; 618 process_done = true; 619 620 // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread 621 _mi_prim_thread_done_auto_done(); 622 623 #ifndef MI_SKIP_COLLECT_ON_EXIT 624 #if (MI_DEBUG || !defined(MI_SHARED_LIB)) 625 // free all memory if possible on process exit. This is not needed for a stand-alone process 626 // but should be done if mimalloc is statically linked into another shared library which 627 // is repeatedly loaded/unloaded, see issue #281. 628 mi_collect(true /* force */ ); 629 #endif 630 #endif 631 632 // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free 633 // since after process_done there might still be other code running that calls `free` (like at_exit routines, 634 // or C-runtime termination code. 635 if (mi_option_is_enabled(mi_option_destroy_on_exit)) { 636 mi_collect(true /* force */); 637 _mi_heap_unsafe_destroy_all(); // forcefully release all memory held by all heaps (of this thread only!) 638 _mi_arena_unsafe_destroy_all(& _mi_heap_main_get()->tld->stats); 639 } 640 641 if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { 642 mi_stats_print(NULL); 643 } 644 _mi_allocator_done(); 645 _mi_verbose_message("process done: 0x%zx\n", _mi_heap_main.thread_id); 646 os_preloading = true; // don't call the C runtime anymore 647 } 648