mimalloc.h
1 /* ---------------------------------------------------------------------------- 2 Copyright (c) 2018-2023, Microsoft Research, Daan Leijen 3 This is free software; you can redistribute it and/or modify it under the 4 terms of the MIT license. A copy of the license can be found in the file 5 "LICENSE" at the root of this distribution. 6 -----------------------------------------------------------------------------*/ 7 #pragma once 8 #ifndef MIMALLOC_H 9 #define MIMALLOC_H 10 11 #define MI_MALLOC_VERSION 188 // major + 2 digits minor 12 13 // ------------------------------------------------------ 14 // Compiler specific attributes 15 // ------------------------------------------------------ 16 17 #ifdef __cplusplus 18 #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 19 #define mi_attr_noexcept noexcept 20 #else 21 #define mi_attr_noexcept throw() 22 #endif 23 #else 24 #define mi_attr_noexcept 25 #endif 26 27 #if defined(__cplusplus) && (__cplusplus >= 201703) 28 #define mi_decl_nodiscard [[nodiscard]] 29 #elif (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // includes clang, icc, and clang-cl 30 #define mi_decl_nodiscard __attribute__((warn_unused_result)) 31 #elif defined(_HAS_NODISCARD) 32 #define mi_decl_nodiscard _NODISCARD 33 #elif (_MSC_VER >= 1700) 34 #define mi_decl_nodiscard _Check_return_ 35 #else 36 #define mi_decl_nodiscard 37 #endif 38 39 #if defined(_MSC_VER) || defined(__MINGW32__) 40 #if !defined(MI_SHARED_LIB) 41 #define mi_decl_export 42 #elif defined(MI_SHARED_LIB_EXPORT) 43 #define mi_decl_export __declspec(dllexport) 44 #else 45 #define mi_decl_export __declspec(dllimport) 46 #endif 47 #if defined(__MINGW32__) 48 #define mi_decl_restrict 49 #define mi_attr_malloc __attribute__((malloc)) 50 #else 51 #if (_MSC_VER >= 1900) && !defined(__EDG__) 52 #define mi_decl_restrict __declspec(allocator) __declspec(restrict) 53 #else 54 #define mi_decl_restrict __declspec(restrict) 55 #endif 56 #define mi_attr_malloc 57 #endif 58 #define mi_cdecl __cdecl 59 #define mi_attr_alloc_size(s) 60 #define mi_attr_alloc_size2(s1,s2) 61 #define mi_attr_alloc_align(p) 62 #elif defined(__GNUC__) // includes clang and icc 63 #if defined(MI_SHARED_LIB) && defined(MI_SHARED_LIB_EXPORT) 64 #define mi_decl_export __attribute__((visibility("default"))) 65 #else 66 #define mi_decl_export 67 #endif 68 #define mi_cdecl // leads to warnings... __attribute__((cdecl)) 69 #define mi_decl_restrict 70 #define mi_attr_malloc __attribute__((malloc)) 71 #if (defined(__clang_major__) && (__clang_major__ < 4)) || (__GNUC__ < 5) 72 #define mi_attr_alloc_size(s) 73 #define mi_attr_alloc_size2(s1,s2) 74 #define mi_attr_alloc_align(p) 75 #elif defined(__INTEL_COMPILER) 76 #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) 77 #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) 78 #define mi_attr_alloc_align(p) 79 #else 80 #define mi_attr_alloc_size(s) __attribute__((alloc_size(s))) 81 #define mi_attr_alloc_size2(s1,s2) __attribute__((alloc_size(s1,s2))) 82 #define mi_attr_alloc_align(p) __attribute__((alloc_align(p))) 83 #endif 84 #else 85 #define mi_cdecl 86 #define mi_decl_export 87 #define mi_decl_restrict 88 #define mi_attr_malloc 89 #define mi_attr_alloc_size(s) 90 #define mi_attr_alloc_size2(s1,s2) 91 #define mi_attr_alloc_align(p) 92 #endif 93 94 // ------------------------------------------------------ 95 // Includes 96 // ------------------------------------------------------ 97 98 #include <stddef.h> // size_t 99 #include <stdbool.h> // bool 100 101 #ifdef __cplusplus 102 extern "C" { 103 #endif 104 105 // ------------------------------------------------------ 106 // Standard malloc interface 107 // ------------------------------------------------------ 108 109 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 110 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); 111 mi_decl_nodiscard mi_decl_export void* mi_realloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); 112 mi_decl_export void* mi_expand(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); 113 114 mi_decl_export void mi_free(void* p) mi_attr_noexcept; 115 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strdup(const char* s) mi_attr_noexcept mi_attr_malloc; 116 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_strndup(const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; 117 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; 118 119 // ------------------------------------------------------ 120 // Extended functionality 121 // ------------------------------------------------------ 122 #define MI_SMALL_WSIZE_MAX (128) 123 #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) 124 125 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 126 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 127 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 128 129 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_mallocn(size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); 130 mi_decl_nodiscard mi_decl_export void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); 131 mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); 132 133 mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; 134 mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; 135 136 137 // ------------------------------------------------------ 138 // Internals 139 // ------------------------------------------------------ 140 141 typedef void (mi_cdecl mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg); 142 mi_decl_export void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg) mi_attr_noexcept; 143 144 typedef void (mi_cdecl mi_output_fun)(const char* msg, void* arg); 145 mi_decl_export void mi_register_output(mi_output_fun* out, void* arg) mi_attr_noexcept; 146 147 typedef void (mi_cdecl mi_error_fun)(int err, void* arg); 148 mi_decl_export void mi_register_error(mi_error_fun* fun, void* arg); 149 150 mi_decl_export void mi_collect(bool force) mi_attr_noexcept; 151 mi_decl_export int mi_version(void) mi_attr_noexcept; 152 mi_decl_export void mi_stats_reset(void) mi_attr_noexcept; 153 mi_decl_export void mi_stats_merge(void) mi_attr_noexcept; 154 mi_decl_export void mi_stats_print(void* out) mi_attr_noexcept; // backward compatibility: `out` is ignored and should be NULL 155 mi_decl_export void mi_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; 156 157 mi_decl_export void mi_process_init(void) mi_attr_noexcept; 158 mi_decl_export void mi_thread_init(void) mi_attr_noexcept; 159 mi_decl_export void mi_thread_done(void) mi_attr_noexcept; 160 mi_decl_export void mi_thread_stats_print_out(mi_output_fun* out, void* arg) mi_attr_noexcept; 161 162 mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, 163 size_t* current_rss, size_t* peak_rss, 164 size_t* current_commit, size_t* peak_commit, size_t* page_faults) mi_attr_noexcept; 165 166 // ------------------------------------------------------------------------------------- 167 // Aligned allocation 168 // Note that `alignment` always follows `size` for consistency with unaligned 169 // allocation, but unfortunately this differs from `posix_memalign` and `aligned_alloc`. 170 // ------------------------------------------------------------------------------------- 171 172 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); 173 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 174 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); 175 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 176 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned(size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2) mi_attr_alloc_align(3); 177 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(1,2); 178 mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); 179 mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); 180 181 182 // ------------------------------------------------------------------------------------- 183 // Heaps: first-class, but can only allocate from the same thread that created it. 184 // ------------------------------------------------------------------------------------- 185 186 struct mi_heap_s; 187 typedef struct mi_heap_s mi_heap_t; 188 189 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new(void); 190 mi_decl_export void mi_heap_delete(mi_heap_t* heap); 191 mi_decl_export void mi_heap_destroy(mi_heap_t* heap); 192 mi_decl_export mi_heap_t* mi_heap_set_default(mi_heap_t* heap); 193 mi_decl_export mi_heap_t* mi_heap_get_default(void); 194 mi_decl_export mi_heap_t* mi_heap_get_backing(void); 195 mi_decl_export void mi_heap_collect(mi_heap_t* heap, bool force) mi_attr_noexcept; 196 197 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 198 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 199 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 200 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 201 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 202 203 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); 204 mi_decl_nodiscard mi_decl_export void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); 205 mi_decl_nodiscard mi_decl_export void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); 206 207 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strdup(mi_heap_t* heap, const char* s) mi_attr_noexcept mi_attr_malloc; 208 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n) mi_attr_noexcept mi_attr_malloc; 209 mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name) mi_attr_noexcept mi_attr_malloc; 210 211 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); 212 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 213 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); 214 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); 215 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3) mi_attr_alloc_align(4); 216 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size2(2, 3); 217 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); 218 mi_decl_nodiscard mi_decl_export void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); 219 220 221 // -------------------------------------------------------------------------------- 222 // Zero initialized re-allocation. 223 // Only valid on memory that was originally allocated with zero initialization too. 224 // e.g. `mi_calloc`, `mi_zalloc`, `mi_zalloc_aligned` etc. 225 // see <https://github.com/microsoft/mimalloc/issues/63#issuecomment-508272992> 226 // -------------------------------------------------------------------------------- 227 228 mi_decl_nodiscard mi_decl_export void* mi_rezalloc(void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(2); 229 mi_decl_nodiscard mi_decl_export void* mi_recalloc(void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); 230 231 mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(2) mi_attr_alloc_align(3); 232 mi_decl_nodiscard mi_decl_export void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(2); 233 mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(2,3) mi_attr_alloc_align(4); 234 mi_decl_nodiscard mi_decl_export void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(2,3); 235 236 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); 237 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size) mi_attr_noexcept mi_attr_alloc_size2(3,4); 238 239 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment) mi_attr_noexcept mi_attr_alloc_size(3) mi_attr_alloc_align(4); 240 mi_decl_nodiscard mi_decl_export void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size(3); 241 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept mi_attr_alloc_size2(3,4) mi_attr_alloc_align(5); 242 mi_decl_nodiscard mi_decl_export void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept mi_attr_alloc_size2(3,4); 243 244 245 // ------------------------------------------------------ 246 // Analysis 247 // ------------------------------------------------------ 248 249 mi_decl_export bool mi_heap_contains_block(mi_heap_t* heap, const void* p); 250 mi_decl_export bool mi_heap_check_owned(mi_heap_t* heap, const void* p); 251 mi_decl_export bool mi_check_owned(const void* p); 252 253 // An area of heap space contains blocks of a single size. 254 typedef struct mi_heap_area_s { 255 void* blocks; // start of the area containing heap blocks 256 size_t reserved; // bytes reserved for this area (virtual) 257 size_t committed; // current available bytes for this area 258 size_t used; // number of allocated blocks 259 size_t block_size; // size in bytes of each block 260 size_t full_block_size; // size in bytes of a full block including padding and metadata. 261 int heap_tag; // heap tag associated with this area 262 } mi_heap_area_t; 263 264 typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); 265 266 mi_decl_export bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); 267 268 // Experimental 269 mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept; 270 mi_decl_nodiscard mi_decl_export bool mi_is_redirected(void) mi_attr_noexcept; 271 272 mi_decl_export int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs) mi_attr_noexcept; 273 mi_decl_export int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs) mi_attr_noexcept; 274 275 mi_decl_export int mi_reserve_os_memory(size_t size, bool commit, bool allow_large) mi_attr_noexcept; 276 mi_decl_export bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node) mi_attr_noexcept; 277 278 mi_decl_export void mi_debug_show_arenas(bool show_inuse, bool show_abandoned, bool show_purge) mi_attr_noexcept; 279 280 // Experimental: heaps associated with specific memory arena's 281 typedef int mi_arena_id_t; 282 mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); 283 mi_decl_export int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; 284 mi_decl_export int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; 285 mi_decl_export bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id) mi_attr_noexcept; 286 287 #if MI_MALLOC_VERSION >= 182 288 // Create a heap that only allocates in the specified arena 289 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id); 290 #endif 291 292 293 // Experimental: allow sub-processes whose memory segments stay separated (and no reclamation between them) 294 // Used for example for separate interpreter's in one process. 295 typedef void* mi_subproc_id_t; 296 mi_decl_export mi_subproc_id_t mi_subproc_main(void); 297 mi_decl_export mi_subproc_id_t mi_subproc_new(void); 298 mi_decl_export void mi_subproc_delete(mi_subproc_id_t subproc); 299 mi_decl_export void mi_subproc_add_current_thread(mi_subproc_id_t subproc); // this should be called right after a thread is created (and no allocation has taken place yet) 300 301 // Experimental: visit abandoned heap areas (from threads that have been terminated) 302 mi_decl_export bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); 303 304 // Experimental: create a new heap with a specified heap tag. Set `allow_destroy` to false to allow the thread 305 // to reclaim abandoned memory (with a compatible heap_tag and arena_id) but in that case `mi_heap_destroy` will 306 // fall back to `mi_heap_delete`. 307 mi_decl_nodiscard mi_decl_export mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id); 308 309 // deprecated 310 mi_decl_export int mi_reserve_huge_os_pages(size_t pages, double max_secs, size_t* pages_reserved) mi_attr_noexcept; 311 312 313 // ------------------------------------------------------ 314 // Convenience 315 // ------------------------------------------------------ 316 317 #define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) 318 #define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) 319 #define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) 320 #define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) 321 #define mi_reallocn_tp(p,tp,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) 322 #define mi_recalloc_tp(p,tp,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) 323 324 #define mi_heap_malloc_tp(hp,tp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) 325 #define mi_heap_zalloc_tp(hp,tp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) 326 #define mi_heap_calloc_tp(hp,tp,n) ((tp*)mi_heap_calloc(hp,n,sizeof(tp))) 327 #define mi_heap_mallocn_tp(hp,tp,n) ((tp*)mi_heap_mallocn(hp,n,sizeof(tp))) 328 #define mi_heap_reallocn_tp(hp,p,tp,n) ((tp*)mi_heap_reallocn(hp,p,n,sizeof(tp))) 329 #define mi_heap_recalloc_tp(hp,p,tp,n) ((tp*)mi_heap_recalloc(hp,p,n,sizeof(tp))) 330 331 332 // ------------------------------------------------------ 333 // Options 334 // ------------------------------------------------------ 335 336 typedef enum mi_option_e { 337 // stable options 338 mi_option_show_errors, // print error messages 339 mi_option_show_stats, // print statistics on termination 340 mi_option_verbose, // print verbose messages 341 // advanced options 342 mi_option_eager_commit, // eager commit segments? (after `eager_commit_delay` segments) (=1) 343 mi_option_arena_eager_commit, // eager commit arenas? Use 2 to enable just on overcommit systems (=2) 344 mi_option_purge_decommits, // should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit) 345 mi_option_allow_large_os_pages, // allow large (2 or 4 MiB) OS pages, implies eager commit. If false, also disables THP for the process. 346 mi_option_reserve_huge_os_pages, // reserve N huge OS pages (1GiB pages) at startup 347 mi_option_reserve_huge_os_pages_at, // reserve huge OS pages at a specific NUMA node 348 mi_option_reserve_os_memory, // reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use `mi_option_get_size`) 349 mi_option_deprecated_segment_cache, 350 mi_option_deprecated_page_reset, 351 mi_option_abandoned_page_purge, // immediately purge delayed purges on thread termination 352 mi_option_deprecated_segment_reset, 353 mi_option_eager_commit_delay, // the first N segments per thread are not eagerly committed (but per page in the segment on demand) 354 mi_option_purge_delay, // memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all. (=10) 355 mi_option_use_numa_nodes, // 0 = use all available numa nodes, otherwise use at most N nodes. 356 mi_option_disallow_os_alloc, // 1 = do not use OS memory for allocation (but only programmatically reserved arenas) 357 mi_option_os_tag, // tag used for OS logging (macOS only for now) (=100) 358 mi_option_max_errors, // issue at most N error messages 359 mi_option_max_warnings, // issue at most N warning messages 360 mi_option_max_segment_reclaim, // max. percentage of the abandoned segments can be reclaimed per try (=10%) 361 mi_option_destroy_on_exit, // if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe 362 mi_option_arena_reserve, // initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use `mi_option_get_size`) 363 mi_option_arena_purge_mult, // multiplier for `purge_delay` for the purging delay for arenas (=10) 364 mi_option_purge_extend_delay, 365 mi_option_abandoned_reclaim_on_free, // allow to reclaim an abandoned segment on a free (=1) 366 mi_option_disallow_arena_alloc, // 1 = do not use arena's for allocation (except if using specific arena id's) 367 mi_option_retry_on_oom, // retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows) 368 mi_option_visit_abandoned, // allow visiting heap blocks from abandoned threads (=0) 369 mi_option_debug_guarded_min, // only used when building with MI_DEBUG_GUARDED: minimal rounded object size for guarded objects (=0) 370 mi_option_debug_guarded_max, // only used when building with MI_DEBUG_GUARDED: maximal rounded object size for guarded objects (=0) 371 _mi_option_last, 372 // legacy option names 373 mi_option_large_os_pages = mi_option_allow_large_os_pages, 374 mi_option_eager_region_commit = mi_option_arena_eager_commit, 375 mi_option_reset_decommits = mi_option_purge_decommits, 376 mi_option_reset_delay = mi_option_purge_delay, 377 mi_option_abandoned_page_reset = mi_option_abandoned_page_purge, 378 mi_option_limit_os_alloc = mi_option_disallow_os_alloc 379 } mi_option_t; 380 381 382 mi_decl_nodiscard mi_decl_export bool mi_option_is_enabled(mi_option_t option); 383 mi_decl_export void mi_option_enable(mi_option_t option); 384 mi_decl_export void mi_option_disable(mi_option_t option); 385 mi_decl_export void mi_option_set_enabled(mi_option_t option, bool enable); 386 mi_decl_export void mi_option_set_enabled_default(mi_option_t option, bool enable); 387 388 mi_decl_nodiscard mi_decl_export long mi_option_get(mi_option_t option); 389 mi_decl_nodiscard mi_decl_export long mi_option_get_clamp(mi_option_t option, long min, long max); 390 mi_decl_nodiscard mi_decl_export size_t mi_option_get_size(mi_option_t option); 391 mi_decl_export void mi_option_set(mi_option_t option, long value); 392 mi_decl_export void mi_option_set_default(mi_option_t option, long value); 393 394 395 // ------------------------------------------------------------------------------------------------------- 396 // "mi" prefixed implementations of various posix, Unix, Windows, and C++ allocation functions. 397 // (This can be convenient when providing overrides of these functions as done in `mimalloc-override.h`.) 398 // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our heap before free-ing. 399 // ------------------------------------------------------------------------------------------------------- 400 401 mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; 402 mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; 403 mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; 404 mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; 405 mi_decl_nodiscard mi_decl_export size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept; 406 407 mi_decl_export int mi_posix_memalign(void** p, size_t alignment, size_t size) mi_attr_noexcept; 408 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_memalign(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); 409 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_valloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 410 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_pvalloc(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 411 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_aligned_alloc(size_t alignment, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(1); 412 413 mi_decl_nodiscard mi_decl_export void* mi_reallocarray(void* p, size_t count, size_t size) mi_attr_noexcept mi_attr_alloc_size2(2,3); 414 mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* p, size_t count, size_t size) mi_attr_noexcept; 415 mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; 416 mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; 417 418 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned short* mi_wcsdup(const unsigned short* s) mi_attr_noexcept mi_attr_malloc; 419 mi_decl_nodiscard mi_decl_export mi_decl_restrict unsigned char* mi_mbsdup(const unsigned char* s) mi_attr_noexcept mi_attr_malloc; 420 mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; 421 mi_decl_export int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name) mi_attr_noexcept; 422 423 mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; 424 mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; 425 mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; 426 427 // The `mi_new` wrappers implement C++ semantics on out-of-memory instead of directly returning `NULL`. 428 // (and call `std::get_new_handler` and potentially raise a `std::bad_alloc` exception). 429 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new(size_t size) mi_attr_malloc mi_attr_alloc_size(1); 430 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); 431 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); 432 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1) mi_attr_alloc_align(2); 433 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_new_n(size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(1, 2); 434 mi_decl_nodiscard mi_decl_export void* mi_new_realloc(void* p, size_t newsize) mi_attr_alloc_size(2); 435 mi_decl_nodiscard mi_decl_export void* mi_new_reallocn(void* p, size_t newcount, size_t size) mi_attr_alloc_size2(2, 3); 436 437 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size_t size) mi_attr_malloc mi_attr_alloc_size(2); 438 mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, size_t count, size_t size) mi_attr_malloc mi_attr_alloc_size2(2, 3); 439 440 #ifdef __cplusplus 441 } 442 #endif 443 444 // --------------------------------------------------------------------------------------------- 445 // Implement the C++ std::allocator interface for use in STL containers. 446 // (note: see `mimalloc-new-delete.h` for overriding the new/delete operators globally) 447 // --------------------------------------------------------------------------------------------- 448 #ifdef __cplusplus 449 450 #include <cstddef> // std::size_t 451 #include <cstdint> // PTRDIFF_MAX 452 #if (__cplusplus >= 201103L) || (_MSC_VER > 1900) // C++11 453 #include <type_traits> // std::true_type 454 #include <utility> // std::forward 455 #endif 456 457 template<class T> struct _mi_stl_allocator_common { 458 typedef T value_type; 459 typedef std::size_t size_type; 460 typedef std::ptrdiff_t difference_type; 461 typedef value_type& reference; 462 typedef value_type const& const_reference; 463 typedef value_type* pointer; 464 typedef value_type const* const_pointer; 465 466 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 467 using propagate_on_container_copy_assignment = std::true_type; 468 using propagate_on_container_move_assignment = std::true_type; 469 using propagate_on_container_swap = std::true_type; 470 template <class U, class ...Args> void construct(U* p, Args&& ...args) { ::new(p) U(std::forward<Args>(args)...); } 471 template <class U> void destroy(U* p) mi_attr_noexcept { p->~U(); } 472 #else 473 void construct(pointer p, value_type const& val) { ::new(p) value_type(val); } 474 void destroy(pointer p) { p->~value_type(); } 475 #endif 476 477 size_type max_size() const mi_attr_noexcept { return (PTRDIFF_MAX/sizeof(value_type)); } 478 pointer address(reference x) const { return &x; } 479 const_pointer address(const_reference x) const { return &x; } 480 }; 481 482 template<class T> struct mi_stl_allocator : public _mi_stl_allocator_common<T> { 483 using typename _mi_stl_allocator_common<T>::size_type; 484 using typename _mi_stl_allocator_common<T>::value_type; 485 using typename _mi_stl_allocator_common<T>::pointer; 486 template <class U> struct rebind { typedef mi_stl_allocator<U> other; }; 487 488 mi_stl_allocator() mi_attr_noexcept = default; 489 mi_stl_allocator(const mi_stl_allocator&) mi_attr_noexcept = default; 490 template<class U> mi_stl_allocator(const mi_stl_allocator<U>&) mi_attr_noexcept { } 491 mi_stl_allocator select_on_container_copy_construction() const { return *this; } 492 void deallocate(T* p, size_type) { mi_free(p); } 493 494 #if (__cplusplus >= 201703L) // C++17 495 mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_new_n(count, sizeof(T))); } 496 mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } 497 #else 498 mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_new_n(count, sizeof(value_type))); } 499 #endif 500 501 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 502 using is_always_equal = std::true_type; 503 #endif 504 }; 505 506 template<class T1,class T2> bool operator==(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return true; } 507 template<class T1,class T2> bool operator!=(const mi_stl_allocator<T1>& , const mi_stl_allocator<T2>& ) mi_attr_noexcept { return false; } 508 509 510 #if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) // C++11 511 #define MI_HAS_HEAP_STL_ALLOCATOR 1 512 513 #include <memory> // std::shared_ptr 514 515 // Common base class for STL allocators in a specific heap 516 template<class T, bool _mi_destroy> struct _mi_heap_stl_allocator_common : public _mi_stl_allocator_common<T> { 517 using typename _mi_stl_allocator_common<T>::size_type; 518 using typename _mi_stl_allocator_common<T>::value_type; 519 using typename _mi_stl_allocator_common<T>::pointer; 520 521 _mi_heap_stl_allocator_common(mi_heap_t* hp) : heap(hp, [](mi_heap_t*) {}) {} /* will not delete nor destroy the passed in heap */ 522 523 #if (__cplusplus >= 201703L) // C++17 524 mi_decl_nodiscard T* allocate(size_type count) { return static_cast<T*>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(T))); } 525 mi_decl_nodiscard T* allocate(size_type count, const void*) { return allocate(count); } 526 #else 527 mi_decl_nodiscard pointer allocate(size_type count, const void* = 0) { return static_cast<pointer>(mi_heap_alloc_new_n(this->heap.get(), count, sizeof(value_type))); } 528 #endif 529 530 #if ((__cplusplus >= 201103L) || (_MSC_VER > 1900)) // C++11 531 using is_always_equal = std::false_type; 532 #endif 533 534 void collect(bool force) { mi_heap_collect(this->heap.get(), force); } 535 template<class U> bool is_equal(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) const { return (this->heap == x.heap); } 536 537 protected: 538 std::shared_ptr<mi_heap_t> heap; 539 template<class U, bool D> friend struct _mi_heap_stl_allocator_common; 540 541 _mi_heap_stl_allocator_common() { 542 mi_heap_t* hp = mi_heap_new(); 543 this->heap.reset(hp, (_mi_destroy ? &heap_destroy : &heap_delete)); /* calls heap_delete/destroy when the refcount drops to zero */ 544 } 545 _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common& x) mi_attr_noexcept : heap(x.heap) { } 546 template<class U> _mi_heap_stl_allocator_common(const _mi_heap_stl_allocator_common<U, _mi_destroy>& x) mi_attr_noexcept : heap(x.heap) { } 547 548 private: 549 static void heap_delete(mi_heap_t* hp) { if (hp != NULL) { mi_heap_delete(hp); } } 550 static void heap_destroy(mi_heap_t* hp) { if (hp != NULL) { mi_heap_destroy(hp); } } 551 }; 552 553 // STL allocator allocation in a specific heap 554 template<class T> struct mi_heap_stl_allocator : public _mi_heap_stl_allocator_common<T, false> { 555 using typename _mi_heap_stl_allocator_common<T, false>::size_type; 556 mi_heap_stl_allocator() : _mi_heap_stl_allocator_common<T, false>() { } // creates fresh heap that is deleted when the destructor is called 557 mi_heap_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, false>(hp) { } // no delete nor destroy on the passed in heap 558 template<class U> mi_heap_stl_allocator(const mi_heap_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, false>(x) { } 559 560 mi_heap_stl_allocator select_on_container_copy_construction() const { return *this; } 561 void deallocate(T* p, size_type) { mi_free(p); } 562 template<class U> struct rebind { typedef mi_heap_stl_allocator<U> other; }; 563 }; 564 565 template<class T1, class T2> bool operator==(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); } 566 template<class T1, class T2> bool operator!=(const mi_heap_stl_allocator<T1>& x, const mi_heap_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); } 567 568 569 // STL allocator allocation in a specific heap, where `free` does nothing and 570 // the heap is destroyed in one go on destruction -- use with care! 571 template<class T> struct mi_heap_destroy_stl_allocator : public _mi_heap_stl_allocator_common<T, true> { 572 using typename _mi_heap_stl_allocator_common<T, true>::size_type; 573 mi_heap_destroy_stl_allocator() : _mi_heap_stl_allocator_common<T, true>() { } // creates fresh heap that is destroyed when the destructor is called 574 mi_heap_destroy_stl_allocator(mi_heap_t* hp) : _mi_heap_stl_allocator_common<T, true>(hp) { } // no delete nor destroy on the passed in heap 575 template<class U> mi_heap_destroy_stl_allocator(const mi_heap_destroy_stl_allocator<U>& x) mi_attr_noexcept : _mi_heap_stl_allocator_common<T, true>(x) { } 576 577 mi_heap_destroy_stl_allocator select_on_container_copy_construction() const { return *this; } 578 void deallocate(T*, size_type) { /* do nothing as we destroy the heap on destruct. */ } 579 template<class U> struct rebind { typedef mi_heap_destroy_stl_allocator<U> other; }; 580 }; 581 582 template<class T1, class T2> bool operator==(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (x.is_equal(y)); } 583 template<class T1, class T2> bool operator!=(const mi_heap_destroy_stl_allocator<T1>& x, const mi_heap_destroy_stl_allocator<T2>& y) mi_attr_noexcept { return (!x.is_equal(y)); } 584 585 #endif // C++11 586 587 #endif // __cplusplus 588 589 #endif